// This implementation targets the RVA23 profile, particularly V, Zvbb, and // Zbb, that is the vector extension and the bit-manipulation extensions. As of // December 2023, most real-world hardware does *not* support these extensions. // This implementation also assumes that misaligned vector loads and stores are // supported, in particular for the vlsseg8e32.v and vssseg8e32.v instructions. // // Compiling and testing this code requires very recent versions of Clang (v17) // and QEMU (v8.2). #define IV0 0x6A09E667 #define IV1 0xBB67AE85 #define IV2 0x3C6EF372 #define IV3 0xA54FF53A // NOTE: Keep this in sync with the same constant in Rust. #define MAX_SIMD_DEGREE 16 #define TRANSPOSED_STRIDE_BYTES 2 * MAX_SIMD_DEGREE * 4 #define CHUNK_START (1 << 0) #define CHUNK_END (1 << 1) #define PARENT (1 << 2) #define ROOT (1 << 3) #define KEYED_HASH (1 << 4) #define DERIVE_KEY_CONTEXT (1 << 5) #define DERIVE_KEY_MATERIAL (1 << 6) .section .text .p2align 2 IV_VEC: .word IV0, IV1, IV2, IV3 ROR1: .word 3, 0, 1, 2 ROR2: .word 2, 3, 0, 1 ROR3: .word 1, 2, 3, 0 # The bottom half of the load permutation is tweaked to account for the fact that # we hold the second row fixed during diagonalization. MSG_LOAD: .short 0, 2, 4, 6, 1, 3, 5, 7, 14, 8, 10, 12, 15, 9, 11, 13 # The message permutation as given in the in the BLAKE3 spec would be the correct # permutation to use if the load order above was 0, 1, 2, 3... However, since # we're using a tricky load order, we need to adjust the permutation accordingly. # The following Python snippet reproduces the permutation we're using here: # # load_order = [0, 2, 4, 6, 1, 3, 5, 7, 14, 8, 10, 12, 15, 9, 11, 13] # original_permutation = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] # retargeted_permutation = [load_order.index(x) for x in original_permutation] # shuffled_permutation = [retargeted_permutation[i] for i in load_order] # print(shuffled_permutation) MSG_PERMUTE: .short 1, 5, 7, 2, 3, 10, 0, 15, 12, 4, 11, 13, 9, 14, 6, 8 // a0: block (zero-padded to 64 bytes) // a1: block_len // a2: cv_bytes // a3: counter // a4: flags // a5: out_ptr .global blake3_guts_riscv_rva23u64_compress blake3_guts_riscv_rva23u64_compress: // Load the message load and message permutation indexes. vsetivli zero, 16, e16, m2, ta, ma la t0, MSG_LOAD vle16.v v8, (t0) la t0, MSG_PERMUTE vle16.v v10, (t0) // Load the CV into v0-v1. vsetivli zero, 16, e8, m1, ta, ma vle8.v v0, (a2) addi a2, a2, 16 vle8.v v1, (a2) // Set LMUL=4 and load the message block temporarily into scratch // space. Apply the MSG_LOAD permutation, and then move the permuted // message words into v4-v7. // TODO: Do this with less register movement? li t0, 64 vsetvli zero, t0, e8, m4, ta, ma vle8.v v20, (a0) vsetivli zero, 16, e32, m4, ta, ma vrgatherei16.vv v16, v20, v8 vsetivli zero, 4, e32, m4, ta, ma vslidedown.vi v20, v16, 4 vslidedown.vi v24, v16, 8 vslidedown.vi v28, v16, 12 vsetivli zero, 4, e32, m1, ta, ma vmv.v.v v4, v16 vmv.v.v v5, v20 vmv.v.v v6, v24 vmv.v.v v7, v28 // Load the diagonalization gather indexes. la t0, ROR1 vle32.v v12, (t0) la t0, ROR2 vle32.v v13, (t0) la t0, ROR3 vle32.v v14, (t0) // Load the IV words. la t0, IV_VEC vle32.v v2, (t0) // Load the counter, block_len, and flags. vsetivli zero, 4, e32, m1, ta, ma vslide1down.vx v3, v3, a3 srli a3, a3, 32 vslide1down.vx v3, v3, a3 vslide1down.vx v3, v3, a1 vslide1down.vx v3, v3, a4 li t0, 7 // round counter blake3_guts_riscv_rva23u64_compress_round_loop: vadd.vv v0, v0, v4 vadd.vv v0, v0, v1 vxor.vv v3, v3, v0 vror.vi v3, v3, 16 vadd.vv v2, v2, v3 vxor.vv v1, v1, v2 vror.vi v1, v1, 12 vadd.vv v0, v0, v5 vadd.vv v0, v0, v1 vxor.vv v3, v3, v0 vror.vi v3, v3, 8 vadd.vv v2, v2, v3 vxor.vv v1, v1, v2 vror.vi v1, v1, 7 // Gathers can't overlap a source register, so use v20/v22/v23 in place // of v0/v2/v3 for this section. vrgather.vv v20, v0, v12 vrgather.vv v23, v3, v13 vrgather.vv v22, v2, v14 vadd.vv v20, v20, v6 vadd.vv v20, v20, v1 vxor.vv v23, v23, v20 vror.vi v23, v23, 16 vadd.vv v22, v22, v23 vxor.vv v1, v1, v22 vror.vi v1, v1, 12 vadd.vv v20, v20, v7 vadd.vv v20, v20, v1 vxor.vv v23, v23, v20 vror.vi v23, v23, 8 vadd.vv v22, v22, v23 vxor.vv v1, v1, v22 vror.vi v1, v1, 7 vrgather.vv v0, v20, v14 vrgather.vv v3, v23, v13 vrgather.vv v2, v22, v12 addi t0, t0, -1 beqz t0, blake3_guts_riscv_rva23u64_compress_end // Shuffle message words. // TODO: Find a way to do this without so much movement? vmv.v.v v16, v4 vmv.v.v v20, v5 vmv.v.v v24, v6 vmv.v.v v28, v7 vsetivli zero, 16, e32, m4, ta, ma vslideup.vi v16, v20, 4 vslideup.vi v16, v24, 8 vslideup.vi v16, v28, 12 vrgatherei16.vv v28, v16, v10 vsetivli zero, 4, e32, m4, ta, ma vslidedown.vi v16, v28, 4 vslidedown.vi v20, v28, 8 vslidedown.vi v24, v28, 12 vsetivli zero, 4, e32, m1, ta, ma vmv.v.v v4, v28 vmv.v.v v5, v16 vmv.v.v v6, v20 vmv.v.v v7, v24 j blake3_guts_riscv_rva23u64_compress_round_loop blake3_guts_riscv_rva23u64_compress_end: vxor.vv v0, v0, v2 vxor.vv v1, v1, v3 vsetivli zero, 16, e8, m1, ta, ma vse8.v v0, (a5) addi a5, a5, 16 vse8.v v1, (a5) ret .global blake3_guts_riscv_rva23u64_degree blake3_guts_riscv_rva23u64_degree: csrr t0, vlenb srli t0, t0, 2 li t1, MAX_SIMD_DEGREE minu a0, t0, t1 ret // clobbers: t0 blake3_guts_riscv_rva23u64_kernel: li t0, IV0 vmv.v.x v8, t0 li t0, IV1 vmv.v.x v9, t0 li t0, IV2 vmv.v.x v10, t0 li t0, IV3 vmv.v.x v11, t0 vadd.vv v0, v0, v16 vadd.vv v1, v1, v18 vadd.vv v2, v2, v20 vadd.vv v3, v3, v22 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vror.vi v15, v15, 16 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 12 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vadd.vv v0, v0, v17 vadd.vv v1, v1, v19 vadd.vv v2, v2, v21 vadd.vv v3, v3, v23 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vror.vi v15, v15, 8 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 7 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vadd.vv v0, v0, v24 vadd.vv v1, v1, v26 vadd.vv v2, v2, v28 vadd.vv v3, v3, v30 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 16 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vror.vi v4, v4, 12 vadd.vv v0, v0, v25 vadd.vv v1, v1, v27 vadd.vv v2, v2, v29 vadd.vv v3, v3, v31 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 8 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vror.vi v4, v4, 7 vadd.vv v0, v0, v18 vadd.vv v1, v1, v19 vadd.vv v2, v2, v23 vadd.vv v3, v3, v20 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vror.vi v15, v15, 16 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 12 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vadd.vv v0, v0, v22 vadd.vv v1, v1, v26 vadd.vv v2, v2, v16 vadd.vv v3, v3, v29 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vror.vi v15, v15, 8 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 7 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vadd.vv v0, v0, v17 vadd.vv v1, v1, v28 vadd.vv v2, v2, v25 vadd.vv v3, v3, v31 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 16 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vror.vi v4, v4, 12 vadd.vv v0, v0, v27 vadd.vv v1, v1, v21 vadd.vv v2, v2, v30 vadd.vv v3, v3, v24 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 8 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vror.vi v4, v4, 7 vadd.vv v0, v0, v19 vadd.vv v1, v1, v26 vadd.vv v2, v2, v29 vadd.vv v3, v3, v23 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vror.vi v15, v15, 16 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 12 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vadd.vv v0, v0, v20 vadd.vv v1, v1, v28 vadd.vv v2, v2, v18 vadd.vv v3, v3, v30 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vror.vi v15, v15, 8 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 7 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vadd.vv v0, v0, v22 vadd.vv v1, v1, v25 vadd.vv v2, v2, v27 vadd.vv v3, v3, v24 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 16 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vror.vi v4, v4, 12 vadd.vv v0, v0, v21 vadd.vv v1, v1, v16 vadd.vv v2, v2, v31 vadd.vv v3, v3, v17 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 8 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vror.vi v4, v4, 7 vadd.vv v0, v0, v26 vadd.vv v1, v1, v28 vadd.vv v2, v2, v30 vadd.vv v3, v3, v29 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vror.vi v15, v15, 16 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 12 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vadd.vv v0, v0, v23 vadd.vv v1, v1, v25 vadd.vv v2, v2, v19 vadd.vv v3, v3, v31 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vror.vi v15, v15, 8 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 7 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vadd.vv v0, v0, v20 vadd.vv v1, v1, v27 vadd.vv v2, v2, v21 vadd.vv v3, v3, v17 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 16 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vror.vi v4, v4, 12 vadd.vv v0, v0, v16 vadd.vv v1, v1, v18 vadd.vv v2, v2, v24 vadd.vv v3, v3, v22 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 8 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vror.vi v4, v4, 7 vadd.vv v0, v0, v28 vadd.vv v1, v1, v25 vadd.vv v2, v2, v31 vadd.vv v3, v3, v30 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vror.vi v15, v15, 16 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 12 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vadd.vv v0, v0, v29 vadd.vv v1, v1, v27 vadd.vv v2, v2, v26 vadd.vv v3, v3, v24 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vror.vi v15, v15, 8 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 7 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vadd.vv v0, v0, v23 vadd.vv v1, v1, v21 vadd.vv v2, v2, v16 vadd.vv v3, v3, v22 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 16 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vror.vi v4, v4, 12 vadd.vv v0, v0, v18 vadd.vv v1, v1, v19 vadd.vv v2, v2, v17 vadd.vv v3, v3, v20 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 8 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vror.vi v4, v4, 7 vadd.vv v0, v0, v25 vadd.vv v1, v1, v27 vadd.vv v2, v2, v24 vadd.vv v3, v3, v31 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vror.vi v15, v15, 16 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 12 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vadd.vv v0, v0, v30 vadd.vv v1, v1, v21 vadd.vv v2, v2, v28 vadd.vv v3, v3, v17 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vror.vi v15, v15, 8 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 7 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vadd.vv v0, v0, v29 vadd.vv v1, v1, v16 vadd.vv v2, v2, v18 vadd.vv v3, v3, v20 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 16 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vror.vi v4, v4, 12 vadd.vv v0, v0, v19 vadd.vv v1, v1, v26 vadd.vv v2, v2, v22 vadd.vv v3, v3, v23 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 8 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vror.vi v4, v4, 7 vadd.vv v0, v0, v27 vadd.vv v1, v1, v21 vadd.vv v2, v2, v17 vadd.vv v3, v3, v24 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vror.vi v15, v15, 16 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 12 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vadd.vv v0, v0, v31 vadd.vv v1, v1, v16 vadd.vv v2, v2, v25 vadd.vv v3, v3, v22 vadd.vv v0, v0, v4 vadd.vv v1, v1, v5 vadd.vv v2, v2, v6 vadd.vv v3, v3, v7 vxor.vv v12, v12, v0 vxor.vv v13, v13, v1 vxor.vv v14, v14, v2 vxor.vv v15, v15, v3 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vror.vi v15, v15, 8 vadd.vv v8, v8, v12 vadd.vv v9, v9, v13 vadd.vv v10, v10, v14 vadd.vv v11, v11, v15 vxor.vv v4, v4, v8 vxor.vv v5, v5, v9 vxor.vv v6, v6, v10 vxor.vv v7, v7, v11 vror.vi v4, v4, 7 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vadd.vv v0, v0, v30 vadd.vv v1, v1, v18 vadd.vv v2, v2, v19 vadd.vv v3, v3, v23 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 16 vror.vi v12, v12, 16 vror.vi v13, v13, 16 vror.vi v14, v14, 16 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 12 vror.vi v6, v6, 12 vror.vi v7, v7, 12 vror.vi v4, v4, 12 vadd.vv v0, v0, v26 vadd.vv v1, v1, v28 vadd.vv v2, v2, v20 vadd.vv v3, v3, v29 vadd.vv v0, v0, v5 vadd.vv v1, v1, v6 vadd.vv v2, v2, v7 vadd.vv v3, v3, v4 vxor.vv v15, v15, v0 vxor.vv v12, v12, v1 vxor.vv v13, v13, v2 vxor.vv v14, v14, v3 vror.vi v15, v15, 8 vror.vi v12, v12, 8 vror.vi v13, v13, 8 vror.vi v14, v14, 8 vadd.vv v10, v10, v15 vadd.vv v11, v11, v12 vadd.vv v8, v8, v13 vadd.vv v9, v9, v14 vxor.vv v5, v5, v10 vxor.vv v6, v6, v11 vxor.vv v7, v7, v8 vxor.vv v4, v4, v9 vror.vi v5, v5, 7 vror.vi v6, v6, 7 vror.vi v7, v7, 7 vror.vi v4, v4, 7 ret // arguments from hash_chunks // a0: input [adjusted by 64] // a1: input_len [adjusted by -64] // a2: key [unused] // a3: counter // a4: flags // a5: aligned+transposed output [unused] // a6: total chunks [unused] // a7: remaining_bytes_in_last_chunk blake3_guts_riscv_rva23u64_hash_blocks: // t0 := full_blocks := (input_len + 1024 - 64) / 1024 addi t0, a1, 1024 - 64 srli t0, t0, 10 // Load and transpose full message blocks. These are "strided segment // loads". Each vlsseg8e32 instruction transposes 8 words from multiple // message blocks into 8 registers, so we need two vlsseg8e32 // instructions (with the second offset by 32 bytes) to load full // 64-byte blocks. The 1024-byte stride represents the spacing between // two blocks in the same position in adjacent chunks. // NOTE: If the final chunk is short, this could be 1 less than the // total number of chunks, in which case this setup code and the kernel // will leave a CV word undisturbed in each of v0-v7. // NOTE: These loads could be misaligned. As far as I know, the Linux // RISC-V ABI allows misaligned loads and stores. If we need to support // an environment that doesn't allow them (or where they're // unacceptably slow), we could add a fallback here. vsetvli zero, t0, e32, m1, ta, ma li t1, 1024 addi t2, a0, 32 vlsseg8e32.v v16, (a0), t1 vlsseg8e32.v v24, (t2), t1 // If remaining_bytes_in_last_chunk in 1..=63, there's a partial block // at the end. Handle it out-of-line. If we take this branch, it will // increment t0 by 1. addi t1, a7, -1 li t2, 63 bltu t1, t2, handle_partial_block partial_block_finished: // load the counter vsetvli zero, t0, e64, m2, ta, ma vmv.v.x v8, a3 vid.v v10 vadd.vv v8, v8, v10 // This is the mode setting that the kernel will use. If the final // chunk is short, this iteration might have fewer blocks than an // earlier iteration, so we need the tail undisturbed (tu). vsetvli zero, t0, e32, m1, tu, ma vncvt.x.x.w v12, v8 li t1, 32 vnsrl.wx v13, v8, t1 // Broadcast the block length, then overwrite the last block's length // to be ((min(64, remaining_bytes_in_last_chunk) - 1) % 64) + 1. That // is: 64 if remaining_bytes_in_last_chunk >= 64 // else 64 if remaining_bytes_in_last_chunk is 0 // else remaining_bytes_in_last_chunk li t1, 64 vmv.v.x v14, t1 minu t1, t1, a7 addi t1, t1, -1 andi t1, t1, 63 addi t1, t1, 1 vslide1down.vx v14, v14, t1 // Broadcast the flags, then set CHUNK_END in the last block's flags if // remaining_bytes_in_last_chunk is in 1..=64. vmv.v.x v15, a4 addi t1, a7, -1 sltiu t1, t1, 64 slli t1, t1, 1 // CHUNK_END = 2 or t1, t1, a4 vslide1down.vx v15, v15, t1 // execute the kernel mv t6, ra call blake3_guts_riscv_rva23u64_kernel mv ra, t6 // xor the two halves of the state vxor.vv v0, v0, v8 vxor.vv v1, v1, v9 vxor.vv v2, v2, v10 vxor.vv v3, v3, v11 vxor.vv v4, v4, v12 vxor.vv v5, v5, v13 vxor.vv v6, v6, v14 vxor.vv v7, v7, v15 // Increment the input pointer, input_len, and // remaining_bytes_in_last_chunk (which cannot go below zero). addi a0, a0, 64 addi a1, a1, -64 addi a7, a7, -64 max a7, a7, zero ret handle_partial_block: // The minimum VLEN is 128 bits, so we're guaranteed to be able to fit // the block in v8-v11 with LMUL=4. Clear 64 zero bytes before the // load, to make sure the partial block is zero-padded. li t1, 64 vsetvli zero, t1, e8, m4, ta, ma vmv.v.i v8, 0 add t2, a0, a1 sub t2, t2, a7 vsetvli zero, a7, e8, m4, ta, ma vle8.v v8, (t2) // If VLEN is longer than 128 bits (16 bytes), then half or all of the // block bytes will be in v8. Make sure they're split evenly across // v8-v11. csrr t1, vlenb li t2, 64 bltu t1, t2, vlenb_less_than_64 vsetivli zero, 8, e32, m1, ta, ma vslidedown.vi v9, v8, 8 vlenb_less_than_64: li t2, 32 bltu t1, t2, vlenb_less_than_32 vsetivli zero, 4, e32, m1, ta, ma vmv.v.v v10, v9 vslidedown.vi v11, v9, 4 vslidedown.vi v9, v8, 4 vlenb_less_than_32: // Shift each of the words of the padded partial block to the end of // the corresponding message vector. t0 was previously the number of // full blocks. Now we increment it, so that it's the number of all // blocks (both full and partial). mv t1, t0 addi t0, t0, 1 // Set vl to at least 4, because v8-v11 each have 4 message words. // Setting vl shorter will make vslide1down clobber those words. li t2, 4 maxu t2, t0, t2 vsetvli zero, t2, e32, m1, ta, ma vslideup.vx v16, v8, t1 vslide1down.vx v8, v8, zero vslideup.vx v17, v8, t1 vslide1down.vx v8, v8, zero vslideup.vx v18, v8, t1 vslide1down.vx v8, v8, zero vslideup.vx v19, v8, t1 vslideup.vx v20, v9, t1 vslide1down.vx v9, v9, zero vslideup.vx v21, v9, t1 vslide1down.vx v9, v9, zero vslideup.vx v22, v9, t1 vslide1down.vx v9, v9, zero vslideup.vx v23, v9, t1 vslideup.vx v24, v10, t1 vslide1down.vx v10, v10, zero vslideup.vx v25, v10, t1 vslide1down.vx v10, v10, zero vslideup.vx v26, v10, t1 vslide1down.vx v10, v10, zero vslideup.vx v27, v10, t1 vslideup.vx v28, v11, t1 vslide1down.vx v11, v11, zero vslideup.vx v29, v11, t1 vslide1down.vx v11, v11, zero vslideup.vx v30, v11, t1 vslide1down.vx v11, v11, zero vslideup.vx v31, v11, t1 j partial_block_finished // a0: input // a1: input_len // a2: key // a3: counter // a4: flags // a5: aligned+transposed output .global blake3_guts_riscv_rva23u64_hash_chunks blake3_guts_riscv_rva23u64_hash_chunks: // Save the original num_chunks = (input_len+1023)/1024 in a6. addi a6, a1, 1023 srli a6, a6, 10 // Track the bytes remaining in the last chunk in a7. The initial value // of this is ((input_len - 1) % 1024) + 1. (The input to this function // is never empty.) It decrements by 64 with each call to // blake3_guts_riscv_rva23u64_hash_chunks, but not below 0. addi a7, a1, -1 andi a7, a7, 1023 addi a7, a7, 1 // broadcast the key to v0-7 vsetvli zero, a6, e32, m1, ta, ma lw t0, 0(a2) vmv.v.x v0, t0 lw t0, 4(a2) vmv.v.x v1, t0 lw t0, 8(a2) vmv.v.x v2, t0 lw t0, 12(a2) vmv.v.x v3, t0 lw t0, 16(a2) vmv.v.x v4, t0 lw t0, 20(a2) vmv.v.x v5, t0 lw t0, 24(a2) vmv.v.x v6, t0 lw t0, 28(a2) vmv.v.x v7, t0 // sixteen blocks (TODO: partial chunks) // Note that hash_blocks increments the input pointer and decrements // the input length. mv t5, ra ori a4, a4, 1 // set CHUNK_START call blake3_guts_riscv_rva23u64_hash_blocks andi a4, a4, -2 // unset CHUNK_START call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks call blake3_guts_riscv_rva23u64_hash_blocks ori a4, a4, 2 // set CHUNK_END call blake3_guts_riscv_rva23u64_hash_blocks mv ra, t5 // If the final chunk is short, we need to set vl back to the total // number of chunks. vsetvli zero, a6, e32, m1, ta, ma // write aligned+transposed outputs with a stride of 2*MAX_SIMD_DEGREE words vse32.v v0, (a5) addi a5, a5, TRANSPOSED_STRIDE_BYTES vse32.v v1, (a5) addi a5, a5, TRANSPOSED_STRIDE_BYTES vse32.v v2, (a5) addi a5, a5, TRANSPOSED_STRIDE_BYTES vse32.v v3, (a5) addi a5, a5, TRANSPOSED_STRIDE_BYTES vse32.v v4, (a5) addi a5, a5, TRANSPOSED_STRIDE_BYTES vse32.v v5, (a5) addi a5, a5, TRANSPOSED_STRIDE_BYTES vse32.v v6, (a5) addi a5, a5, TRANSPOSED_STRIDE_BYTES vse32.v v7, (a5) ret // a0: aligned+transposed input // a1: num_parents // a2: key // a3: flags // a4: out pointer .global blake3_guts_riscv_rva23u64_hash_parents blake3_guts_riscv_rva23u64_hash_parents: // load the transposed CVs and split alternating words into the low and // high halves of the input vectors vsetvli zero, a1, e32, m1, ta, ma vlseg2e32.v v16, (a0) vmv.v.v v24, v17 addi a0, a0, TRANSPOSED_STRIDE_BYTES vlseg2e32.v v17, (a0) vmv.v.v v25, v18 addi a0, a0, TRANSPOSED_STRIDE_BYTES vlseg2e32.v v18, (a0) vmv.v.v v26, v19 addi a0, a0, TRANSPOSED_STRIDE_BYTES vlseg2e32.v v19, (a0) vmv.v.v v27, v20 addi a0, a0, TRANSPOSED_STRIDE_BYTES vlseg2e32.v v20, (a0) vmv.v.v v28, v21 addi a0, a0, TRANSPOSED_STRIDE_BYTES vlseg2e32.v v21, (a0) vmv.v.v v29, v22 addi a0, a0, TRANSPOSED_STRIDE_BYTES vlseg2e32.v v22, (a0) vmv.v.v v30, v23 addi a0, a0, TRANSPOSED_STRIDE_BYTES vlseg2e32.v v14, (a0) // use v14-15 as scratch space to avoid overwriting v24 vmv.v.v v23, v14 vmv.v.v v31, v15 // broadcast the key to v0-7 lw t0, 0(a2) vmv.v.x v0, t0 lw t0, 4(a2) vmv.v.x v1, t0 lw t0, 8(a2) vmv.v.x v2, t0 lw t0, 12(a2) vmv.v.x v3, t0 lw t0, 16(a2) vmv.v.x v4, t0 lw t0, 20(a2) vmv.v.x v5, t0 lw t0, 24(a2) vmv.v.x v6, t0 lw t0, 28(a2) vmv.v.x v7, t0 // zero the counter vmv.v.i v12, 0 vmv.v.i v13, 0 // broadcast the block length li t0, 64 vmv.v.x v14, t0 // broadcast the flags vmv.v.x v15, a3 // execute the kernel mv t6, ra call blake3_guts_riscv_rva23u64_kernel mv ra, t6 // xor the two halves of the state vxor.vv v0, v0, v8 vxor.vv v1, v1, v9 vxor.vv v2, v2, v10 vxor.vv v3, v3, v11 vxor.vv v4, v4, v12 vxor.vv v5, v5, v13 vxor.vv v6, v6, v14 vxor.vv v7, v7, v15 // write aligned+transposed outputs with a stride of 2*MAX_SIMD_DEGREE words vse32.v v0, (a4) addi a4, a4, TRANSPOSED_STRIDE_BYTES vse32.v v1, (a4) addi a4, a4, TRANSPOSED_STRIDE_BYTES vse32.v v2, (a4) addi a4, a4, TRANSPOSED_STRIDE_BYTES vse32.v v3, (a4) addi a4, a4, TRANSPOSED_STRIDE_BYTES vse32.v v4, (a4) addi a4, a4, TRANSPOSED_STRIDE_BYTES vse32.v v5, (a4) addi a4, a4, TRANSPOSED_STRIDE_BYTES vse32.v v6, (a4) addi a4, a4, TRANSPOSED_STRIDE_BYTES vse32.v v7, (a4) ret // a0: 64 zero-padded block bytes // a1: block_len // a2: cv // a3: counter // a4: flags // a5: out_ptr // a6: out_len blake3_guts_riscv_rva23u64_xof_inner: // t1 := total_blocks := (out_len + 63) / 64 addi t1, a6, 63 srli t1, t1, 6 // t2 := full_blocks := out_len / 64 srli t2, a6, 6 // broadcast the CV to v0-7 vsetvli zero, t1, e32, m1, ta, ma lw t3, 0(a2) vmv.v.x v0, t3 lw t3, 4(a2) vmv.v.x v1, t3 lw t3, 8(a2) vmv.v.x v2, t3 lw t3, 12(a2) vmv.v.x v3, t3 lw t3, 16(a2) vmv.v.x v4, t3 lw t3, 20(a2) vmv.v.x v5, t3 lw t3, 24(a2) vmv.v.x v6, t3 lw t3, 28(a2) vmv.v.x v7, t3 // broadcast the block_words to v16-31 lw t3, 0(a0) vmv.v.x v16, t3 lw t3, 4(a0) vmv.v.x v17, t3 lw t3, 8(a0) vmv.v.x v18, t3 lw t3, 12(a0) vmv.v.x v19, t3 lw t3, 16(a0) vmv.v.x v20, t3 lw t3, 20(a0) vmv.v.x v21, t3 lw t3, 24(a0) vmv.v.x v22, t3 lw t3, 28(a0) vmv.v.x v23, t3 lw t3, 32(a0) vmv.v.x v24, t3 lw t3, 36(a0) vmv.v.x v25, t3 lw t3, 40(a0) vmv.v.x v26, t3 lw t3, 44(a0) vmv.v.x v27, t3 lw t3, 48(a0) vmv.v.x v28, t3 lw t3, 52(a0) vmv.v.x v29, t3 lw t3, 56(a0) vmv.v.x v30, t3 lw t3, 60(a0) vmv.v.x v31, t3 // load the counter vsetvli zero, t1, e64, m2, ta, ma vmv.v.x v8, a3 vid.v v10 vadd.vv v8, v8, v10 vsetvli zero, t1, e32, m1, ta, ma vncvt.x.x.w v12, v8 li t3, 32 vnsrl.wx v13, v8, t3 // broadcast the block length vmv.v.x v14, a1 // broadcast the flags vmv.v.x v15, a4 // execute the kernel mv t6, ra call blake3_guts_riscv_rva23u64_kernel mv ra, t6 // reload the CV, this time into v16-23 lw t3, 0(a2) vmv.v.x v16, t3 lw t3, 4(a2) vmv.v.x v17, t3 lw t3, 8(a2) vmv.v.x v18, t3 lw t3, 12(a2) vmv.v.x v19, t3 lw t3, 16(a2) vmv.v.x v20, t3 lw t3, 20(a2) vmv.v.x v21, t3 lw t3, 24(a2) vmv.v.x v22, t3 lw t3, 28(a2) vmv.v.x v23, t3 // xor the two halves of the state and feed-forward the CV vxor.vv v0, v0, v8 vxor.vv v1, v1, v9 vxor.vv v2, v2, v10 vxor.vv v3, v3, v11 vxor.vv v4, v4, v12 vxor.vv v5, v5, v13 vxor.vv v6, v6, v14 vxor.vv v7, v7, v15 vxor.vv v8, v8, v16 vxor.vv v9, v9, v17 vxor.vv v10, v10, v18 vxor.vv v11, v11, v19 vxor.vv v12, v12, v20 vxor.vv v13, v13, v21 vxor.vv v14, v14, v22 vxor.vv v15, v15, v23 ret // a0: 64 zero-padded block bytes // a1: block_len // a2: cv // a3: counter // a4: flags // a5: out_ptr // a6: out_len .global blake3_guts_riscv_rva23u64_xof blake3_guts_riscv_rva23u64_xof: mv t5, ra call blake3_guts_riscv_rva23u64_xof_inner mv ra, t5 // t1 is now total_blocks, and t2 is full_blocks. Set vl to t2 and the // tail policy to undisturbed. We'll handle full blocks with segmented // stores, and then we'll use a separate branch for a partial final // block, if any. vsetvli zero, t2, e32, m1, tu, ma // Transpose and store full output blocks. These are "strided segment // stores". Each vssseg8e32 instruction transposes 8 words from // adjacent registers into 32 bytes of contiguous output, so we need // two vssseg8e32 instructions to store full 64-byte blocks. We offset // the second store by 32 bytes and use a 64-byte stride. // NOTE: These stores might be misaligned. li t0, 64 addi t3, a5, 32 vssseg8e32.v v0, (a5), t0 vssseg8e32.v v8, (t3), t0 // If full_blocks != partial_blocks, we need to handle the final // partial block. Otherwise, we're done. bne t1, t2, blake3_guts_riscv_rva23u64_xof_partial_block ret blake3_guts_riscv_rva23u64_xof_partial_block: // Collect groups of 4 words in v0, v4, v8, and v12. vsetivli zero, 4, e32, m1, ta, ma vslidedown.vx v0, v0, t2 vslidedown.vx v1, v1, t2 vslideup.vi v0, v1, 1 vslidedown.vx v2, v2, t2 vslideup.vi v0, v2, 2 vslidedown.vx v3, v3, t2 vslideup.vi v0, v3, 3 vslidedown.vx v4, v4, t2 vslidedown.vx v5, v5, t2 vslideup.vi v4, v5, 1 vslidedown.vx v6, v6, t2 vslideup.vi v4, v6, 2 vslidedown.vx v7, v7, t2 vslideup.vi v4, v7, 3 vslidedown.vx v8, v8, t2 vslidedown.vx v9, v9, t2 vslideup.vi v8, v9, 1 vslidedown.vx v10, v10, t2 vslideup.vi v8, v10, 2 vslidedown.vx v11, v11, t2 vslideup.vi v8, v11, 3 vslidedown.vx v12, v12, t2 vslidedown.vx v13, v13, t2 vslideup.vi v12, v13, 1 vslidedown.vx v14, v14, t2 vslideup.vi v12, v14, 2 vslidedown.vx v15, v15, t2 vslideup.vi v12, v15, 3 // Use LMUL=4 to guarantee that one vector register group can hold 16 // words, and collect all 16 words in the v0 group. vsetivli zero, 16, e32, m4, ta, ma vslideup.vi v0, v4, 4 vslideup.vi v0, v8, 8 vslideup.vi v0, v12, 12 // Switch to bytes and write the output. andi t3, a6, 63 add a5, a5, a6 sub a5, a5, t3 vsetvli zero, t3, e8, m4, ta, ma vse8.v v0, (a5) ret // a0: 64 zero-padded block bytes // a1: block_len // a2: cv // a3: counter // a4: flags // a5: out_ptr // a6: out_len .global blake3_guts_riscv_rva23u64_xof_xor blake3_guts_riscv_rva23u64_xof_xor: mv t5, ra call blake3_guts_riscv_rva23u64_xof_inner mv ra, t5 // t1 is now total_blocks, and t2 is full_blocks. Set vl to t2 and the // tail policy to undisturbed. We'll handle full blocks with segmented // stores, and then we'll use a separate branch for a partial final // block, if any. vsetvli zero, t2, e32, m1, tu, ma // Do a transposed load of the caller's buffer, xor that with the state // words, and do a transposed store. These are "strided segment" // loads/stores. Each vlsseg8e32/vssseg8e32 instruction works with // groups of 8 words or 32 bytes, so we need pairs of these // instructions to handle full 64-byte blocks. We offset the second by // 32 bytes and use a 64-byte stride. // NOTE: These accesses might be misaligned. li t0, 64 addi t3, a5, 32 vlsseg8e32.v v16, (a5), t0 vlsseg8e32.v v24, (t3), t0 vxor.vv v0, v0, v16 vxor.vv v1, v1, v17 vxor.vv v2, v2, v18 vxor.vv v3, v3, v19 vxor.vv v4, v4, v20 vxor.vv v5, v5, v21 vxor.vv v6, v6, v22 vxor.vv v7, v7, v23 vxor.vv v8, v8, v24 vxor.vv v9, v9, v25 vxor.vv v10, v10, v26 vxor.vv v11, v11, v27 vxor.vv v12, v12, v28 vxor.vv v13, v13, v29 vxor.vv v14, v14, v30 vxor.vv v15, v15, v31 vssseg8e32.v v0, (a5), t0 vssseg8e32.v v8, (t3), t0 // If full_blocks != partial_blocks, we need to handle the final // partial block. Otherwise, we're done. bne t1, t2, blake3_guts_riscv_rva23u64_xof_xor_partial_block ret blake3_guts_riscv_rva23u64_xof_xor_partial_block: // Collect groups of 4 words in v0, v4, v8, and v12. vsetivli zero, 4, e32, m1, ta, ma vslidedown.vx v0, v0, t2 vslidedown.vx v1, v1, t2 vslideup.vi v0, v1, 1 vslidedown.vx v2, v2, t2 vslideup.vi v0, v2, 2 vslidedown.vx v3, v3, t2 vslideup.vi v0, v3, 3 vslidedown.vx v4, v4, t2 vslidedown.vx v5, v5, t2 vslideup.vi v4, v5, 1 vslidedown.vx v6, v6, t2 vslideup.vi v4, v6, 2 vslidedown.vx v7, v7, t2 vslideup.vi v4, v7, 3 vslidedown.vx v8, v8, t2 vslidedown.vx v9, v9, t2 vslideup.vi v8, v9, 1 vslidedown.vx v10, v10, t2 vslideup.vi v8, v10, 2 vslidedown.vx v11, v11, t2 vslideup.vi v8, v11, 3 vslidedown.vx v12, v12, t2 vslidedown.vx v13, v13, t2 vslideup.vi v12, v13, 1 vslidedown.vx v14, v14, t2 vslideup.vi v12, v14, 2 vslidedown.vx v15, v15, t2 vslideup.vi v12, v15, 3 // Use LMUL=4 to guarantee that one vector register group can hold 16 // words, and collect all 16 words in the v0 group. vsetivli zero, 16, e32, m4, ta, ma vslideup.vi v0, v4, 4 vslideup.vi v0, v8, 8 vslideup.vi v0, v12, 12 // Switch to bytes and read/xor/write the output. andi t3, a6, 63 add a5, a5, a6 sub a5, a5, t3 vsetvli zero, t3, e8, m4, ta, ma vle8.v v4, (a5) vxor.vv v0, v0, v4 vse8.v v0, (a5) ret // a0: input_ptr // a1: input_len // a2: key // a3: counter // a4: out_ptr .global blake3_guts_riscv_rva23u64_universal_hash blake3_guts_riscv_rva23u64_universal_hash: // t0 := full_blocks := input_len / 64 srli t0, a1, 6 // Load and transpose full message blocks. These are "strided segment // loads". Each vlsseg8e32 instruction transposes 8 words from multiple // message blocks into 8 registers, so we need two vlsseg8e32 // instructions (with the second offset by 32 bytes) to load full // 64-byte blocks. The 64-byte stride equals the block size, because in // this case (unlike hash_blocks) the blocks are adjacent. // NOTE: These loads could be misaligned. As far as I know, the Linux // RISC-V ABI allows misaligned loads and stores. If we need to support // an environment that doesn't allow them (or where they're // unacceptably slow), we could add a fallback here. vsetvli zero, t0, e32, m1, ta, ma li t1, 64 addi t2, a0, 32 vlsseg8e32.v v16, (a0), t1 vlsseg8e32.v v24, (t2), t1 // Broadcast the block length. li t1, 64 vmv.v.x v14, t1 // If there's a partial block, handle it in an out-of-line branch. andi t1, a1, 63 bnez t1, universal_hash_handle_partial_block universal_hash_partial_block_finished: // Broadcast the key to v0-7. lw t1, 0(a2) vmv.v.x v0, t1 lw t1, 4(a2) vmv.v.x v1, t1 lw t1, 8(a2) vmv.v.x v2, t1 lw t1, 12(a2) vmv.v.x v3, t1 lw t1, 16(a2) vmv.v.x v4, t1 lw t1, 20(a2) vmv.v.x v5, t1 lw t1, 24(a2) vmv.v.x v6, t1 lw t1, 28(a2) vmv.v.x v7, t1 // Load the counter. vsetvli zero, t0, e64, m2, ta, ma vmv.v.x v8, a3 vid.v v10 vadd.vv v8, v8, v10 vsetvli zero, t0, e32, m1, ta, ma vncvt.x.x.w v12, v8 li t1, 32 vnsrl.wx v13, v8, t1 // Broadcast the flags. li t1, CHUNK_START | CHUNK_END | ROOT | KEYED_HASH vmv.v.x v15, t1 // Execute the kernel. mv t6, ra call blake3_guts_riscv_rva23u64_kernel mv ra, t6 // Finish the first four state vectors. The rest are dropped. vxor.vv v0, v0, v8 vxor.vv v1, v1, v9 vxor.vv v2, v2, v10 vxor.vv v3, v3, v11 // XOR-reduce each vector. vmv.v.i v4, 0 vredxor.vs v0, v0, v4 vredxor.vs v1, v1, v4 vredxor.vs v2, v2, v4 vredxor.vs v3, v3, v4 // Write the output. vmv.x.s t0, v0 sw t0, 0(a4) vmv.x.s t0, v1 sw t0, 4(a4) vmv.x.s t0, v2 sw t0, 8(a4) vmv.x.s t0, v3 sw t0, 12(a4) ret universal_hash_handle_partial_block: // Load the partial block into v8-v11. With LMUL=4, v8 is guaranteed to // hold at least 64 bytes. Zero all 64 bytes first, for block padding. // The block length is already in t1. li t2, 64 vsetvli zero, t2, e8, m4, ta, ma vmv.v.i v8, 0 vsetvli zero, t1, e8, m4, ta, ma add t2, a0, a1 sub t2, t2, t1 vle8.v v8, (t2) // If VLEN is longer than 128 bits (16 bytes), then half or all of the // block bytes will be in v8. Make sure they're split evenly across // v8-v11. csrr t2, vlenb li t3, 64 bltu t2, t3, universal_hash_vlenb_less_than_64 vsetivli zero, 8, e32, m1, ta, ma vslidedown.vi v9, v8, 8 universal_hash_vlenb_less_than_64: li t3, 32 bltu t2, t3, universal_hash_vlenb_less_than_32 vsetivli zero, 4, e32, m1, ta, ma vmv.v.v v10, v9 vslidedown.vi v11, v9, 4 vslidedown.vi v9, v8, 4 universal_hash_vlenb_less_than_32: // Shift each of the words of the padded partial block to the end of // the corresponding message vector. t0 was previously the number of // full blocks. Now we increment it, so that it's the number of all // blocks (both full and partial). mv t2, t0 addi t0, t0, 1 // Set vl to at least 4, because v8-v11 each have 4 message words. // Setting vl shorter will make vslide1down clobber those words. li t3, 4 maxu t3, t0, t3 vsetvli zero, t3, e32, m1, ta, ma vslideup.vx v16, v8, t2 vslide1down.vx v8, v8, zero vslideup.vx v17, v8, t2 vslide1down.vx v8, v8, zero vslideup.vx v18, v8, t2 vslide1down.vx v8, v8, zero vslideup.vx v19, v8, t2 vslideup.vx v20, v9, t2 vslide1down.vx v9, v9, zero vslideup.vx v21, v9, t2 vslide1down.vx v9, v9, zero vslideup.vx v22, v9, t2 vslide1down.vx v9, v9, zero vslideup.vx v23, v9, t2 vslideup.vx v24, v10, t2 vslide1down.vx v10, v10, zero vslideup.vx v25, v10, t2 vslide1down.vx v10, v10, zero vslideup.vx v26, v10, t2 vslide1down.vx v10, v10, zero vslideup.vx v27, v10, t2 vslideup.vx v28, v11, t2 vslide1down.vx v11, v11, zero vslideup.vx v29, v11, t2 vslide1down.vx v11, v11, zero vslideup.vx v30, v11, t2 vslide1down.vx v11, v11, zero vslideup.vx v31, v11, t2 // Set the updated VL. vsetvli zero, t0, e32, m1, ta, ma // Append the final block length, still in t1. vmv.v.x v8, t1 addi t2, t0, -1 vslideup.vx v14, v8, t2 j universal_hash_partial_block_finished