Index: /trunk/src/VBox/Runtime/common/asm/asm-fake.cpp =================================================================== --- /trunk/src/VBox/Runtime/common/asm/asm-fake.cpp (revision 51882) +++ /trunk/src/VBox/Runtime/common/asm/asm-fake.cpp (revision 51883) @@ -461,2 +461,8 @@ } +RTDECL(uint64_t) ASMByteSwapU64(uint64_t u64) +{ + return RT_MAKE_U64_FROM_U8(RT_BYTE8(u64), RT_BYTE7(u64), RT_BYTE6(u64), RT_BYTE5(u64), + RT_BYTE4(u64), RT_BYTE3(u64), RT_BYTE2(u64), RT_BYTE1(u64)); +} + Index: /trunk/src/VBox/Runtime/common/checksum/alt-sha256.cpp =================================================================== --- /trunk/src/VBox/Runtime/common/checksum/alt-sha256.cpp (revision 51882) +++ /trunk/src/VBox/Runtime/common/checksum/alt-sha256.cpp (revision 51883) @@ -33,5 +33,5 @@ /** Enables the unrolled code. */ -#define RTSHA1_UNROLLED 1 +#define RTSHA256_UNROLLED 1 @@ -71,5 +71,5 @@ * Global Variables * *******************************************************************************/ -#ifndef RTSHA1_UNROLLED +#ifndef RTSHA256_UNROLLED /** The K constants */ static uint32_t const g_auKs[] = @@ -92,5 +92,5 @@ UINT32_C(0x90befffa), UINT32_C(0xa4506ceb), UINT32_C(0xbef9a3f7), UINT32_C(0xc67178f2), }; -#endif /* !RTSHA1_UNROLLED */ +#endif /* !RTSHA256_UNROLLED */ @@ -206,5 +206,5 @@ DECLINLINE(void) rtSha256BlockInit(PRTSHA256CONTEXT pCtx, uint8_t const *pbBlock) { -#ifdef RTSHA1_UNROLLED +#ifdef RTSHA256_UNROLLED uint32_t const *puSrc = (uint32_t const *)pbBlock; uint32_t *puW = &pCtx->AltPrivate.auW[0]; @@ -235,8 +235,8 @@ *puW++ = ASMByteSwapU32(*puSrc++); # else - memcpy(puW, puSrc, RTSHA1_BLOCK_SIZE); + memcpy(puW, puSrc, RTSHA256_BLOCK_SIZE); # endif -#else /* !RTSHA1_UNROLLED */ +#else /* !RTSHA256_UNROLLED */ uint32_t const *pu32Block = (uint32_t const *)pbBlock; Assert(!((uintptr_t)pu32Block & 3)); @@ -254,5 +254,5 @@ pCtx->AltPrivate.auW[iWord] = u32; } -#endif /* !RTSHA1_UNROLLED */ +#endif /* !RTSHA256_UNROLLED */ } @@ -265,5 +265,5 @@ DECLINLINE(void) rtSha256BlockInitBuffered(PRTSHA256CONTEXT pCtx) { -#ifdef RTSHA1_UNROLLED +#ifdef RTSHA256_UNROLLED uint32_t *puW = &pCtx->AltPrivate.auW[0]; Assert(!((uintptr_t)puW & 3)); @@ -293,5 +293,5 @@ # endif -#else /* !RTSHA1_UNROLLED */ +#else /* !RTSHA256_UNROLLED */ unsigned iWord; for (iWord = 0; iWord < 16; iWord++) @@ -306,5 +306,5 @@ pCtx->AltPrivate.auW[iWord] = u32; } -#endif /* !RTSHA1_UNROLLED */ +#endif /* !RTSHA256_UNROLLED */ } @@ -328,5 +328,5 @@ uint32_t uH = pCtx->AltPrivate.auH[7]; -#ifdef RTSHA1_UNROLLED +#ifdef RTSHA256_UNROLLED uint32_t *puW = &pCtx->AltPrivate.auW[0]; # define RTSHA256_BODY(a_iWord, a_uK, a_uA, a_uB, a_uC, a_uD, a_uE, a_uF, a_uG, a_uH) \ @@ -380,5 +380,5 @@ UINT32_C(0x90befffa), UINT32_C(0xa4506ceb), UINT32_C(0xbef9a3f7), UINT32_C(0xc67178f2), 56); -#else /* !RTSHA1_UNROLLED */ +#else /* !RTSHA256_UNROLLED */ for (unsigned iWord = 0; iWord < RT_ELEMENTS(pCtx->AltPrivate.auW); iWord++) { @@ -401,5 +401,5 @@ uA = uT1 + uT2; } -#endif /* !RTSHA1_UNROLLED */ +#endif /* !RTSHA256_UNROLLED */ pCtx->AltPrivate.auH[0] += uA; Index: /trunk/src/VBox/Runtime/common/checksum/alt-sha512.cpp =================================================================== --- /trunk/src/VBox/Runtime/common/checksum/alt-sha512.cpp (revision 51882) +++ /trunk/src/VBox/Runtime/common/checksum/alt-sha512.cpp (revision 51883) @@ -32,4 +32,7 @@ #define RTSHA512_BLOCK_SIZE 128U +/** Enables the unrolled code. */ +#define RTSHA512_UNROLLED 1 + /******************************************************************************* @@ -68,4 +71,5 @@ * Global Variables * *******************************************************************************/ +#ifndef RTSHA512_UNROLLED /** The K constants. */ static uint64_t const g_auKs[] = @@ -92,4 +96,5 @@ UINT64_C(0x4cc5d4becb3e42b6), UINT64_C(0x597f299cfc657e2a), UINT64_C(0x5fcb6fab3ad6faec), UINT64_C(0x6c44198c4a475817), }; +#endif /* !RTSHA512_UNROLLED */ @@ -114,7 +119,17 @@ DECL_FORCE_INLINE(uint64_t) rtSha512Ch(uint64_t uX, uint64_t uY, uint64_t uZ) { +#if 1 + /* Optimization that saves one operation and probably a temporary variable. */ + uint64_t uResult = uY; + uResult ^= uZ; + uResult &= uX; + uResult ^= uZ; + return uResult; +#else + /* The original. */ uint64_t uResult = uX & uY; uResult ^= ~uX & uZ; return uResult; +#endif } @@ -123,8 +138,18 @@ DECL_FORCE_INLINE(uint64_t) rtSha512Maj(uint64_t uX, uint64_t uY, uint64_t uZ) { +#if 1 + /* Optimization that save one operation and probably a temporary variable. */ + uint64_t uResult = uY; + uResult ^= uZ; + uResult &= uX; + uResult ^= uY & uZ; + return uResult; +#else + /* The original. */ uint64_t uResult = uX & uY; uResult ^= uX & uZ; uResult ^= uY & uZ; return uResult; +#endif } @@ -182,8 +207,42 @@ * * @param pCtx The SHA-512 context. - * @param pbBlock The block. Must be 32-bit aligned. + * @param pbBlock The block. Must be 64-bit aligned. */ DECLINLINE(void) rtSha512BlockInit(PRTSHA512CONTEXT pCtx, uint8_t const *pbBlock) { +#ifdef RTSHA512_UNROLLED + uint64_t const *puSrc = (uint64_t const *)pbBlock; + uint64_t *puW = &pCtx->AltPrivate.auW[0]; + Assert(!((uintptr_t)puSrc & 7)); + Assert(!((uintptr_t)puW & 7)); + + /* Copy and byte-swap the block. Initializing the rest of the Ws are done + in the processing loop. */ +# ifdef RT_LITTLE_ENDIAN + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); + *puW++ = ASMByteSwapU64(*puSrc++); +# else + memcpy(puW, puSrc, RTSHA512_BLOCK_SIZE); +# endif + +#else /* !RTSHA512_UNROLLED */ + uint64_t const *pu32Block = (uint64_t const *)pbBlock; Assert(!((uintptr_t)pu32Block & 3)); @@ -201,4 +260,5 @@ pCtx->AltPrivate.auW[iWord] = u64; } +#endif /* !RTSHA512_UNROLLED */ } @@ -211,4 +271,34 @@ DECLINLINE(void) rtSha512BlockInitBuffered(PRTSHA512CONTEXT pCtx) { +#ifdef RTSHA512_UNROLLED + uint64_t *puW = &pCtx->AltPrivate.auW[0]; + Assert(!((uintptr_t)puW & 7)); + + /* Do the byte swap if necessary. Initializing the rest of the Ws are done + in the processing loop. */ +# ifdef RT_LITTLE_ENDIAN + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; + *puW = ASMByteSwapU64(*puW); puW++; +# endif + +#else /* !RTSHA512_UNROLLED */ + unsigned iWord; for (iWord = 0; iWord < 16; iWord++) @@ -223,4 +313,5 @@ pCtx->AltPrivate.auW[iWord] = u64; } +#endif /* !RTSHA512_UNROLLED */ } @@ -244,4 +335,70 @@ uint64_t uH = pCtx->AltPrivate.auH[7]; +#ifdef RTSHA512_UNROLLED + uint64_t *puW = &pCtx->AltPrivate.auW[0]; +# define RTSHA512_BODY(a_iWord, a_uK, a_uA, a_uB, a_uC, a_uD, a_uE, a_uF, a_uG, a_uH) \ + do { \ + if ((a_iWord) < 16) \ + a_uH += *puW++; \ + else \ + { \ + uint64_t u64 = puW[-16]; \ + u64 += rtSha512SmallSigma0(puW[-15]); \ + u64 += puW[-7]; \ + u64 += rtSha512SmallSigma1(puW[-2]); \ + if (a_iWord < 80-2) *puW++ = u64; else puW++; \ + a_uH += u64; \ + } \ + \ + a_uH += rtSha512CapitalSigma1(a_uE); \ + a_uH += a_uK; \ + a_uH += rtSha512Ch(a_uE, a_uF, a_uG); \ + a_uD += a_uH; \ + \ + a_uH += rtSha512CapitalSigma0(a_uA); \ + a_uH += rtSha512Maj(a_uA, a_uB, a_uC); \ + } while (0) +# define RTSHA512_EIGHT(a_uK0, a_uK1, a_uK2, a_uK3, a_uK4, a_uK5, a_uK6, a_uK7, a_iFirst) \ + do { \ + RTSHA512_BODY(a_iFirst + 0, a_uK0, uA, uB, uC, uD, uE, uF, uG, uH); \ + RTSHA512_BODY(a_iFirst + 1, a_uK1, uH, uA, uB, uC, uD, uE, uF, uG); \ + RTSHA512_BODY(a_iFirst + 2, a_uK2, uG, uH, uA, uB, uC, uD, uE, uF); \ + RTSHA512_BODY(a_iFirst + 3, a_uK3, uF, uG, uH, uA, uB, uC, uD, uE); \ + RTSHA512_BODY(a_iFirst + 4, a_uK4, uE, uF, uG, uH, uA, uB, uC, uD); \ + RTSHA512_BODY(a_iFirst + 5, a_uK5, uD, uE, uF, uG, uH, uA, uB, uC); \ + RTSHA512_BODY(a_iFirst + 6, a_uK6, uC, uD, uE, uF, uG, uH, uA, uB); \ + RTSHA512_BODY(a_iFirst + 7, a_uK7, uB, uC, uD, uE, uF, uG, uH, uA); \ + } while (0) + RTSHA512_EIGHT(UINT64_C(0x428a2f98d728ae22), UINT64_C(0x7137449123ef65cd), UINT64_C(0xb5c0fbcfec4d3b2f), UINT64_C(0xe9b5dba58189dbbc), + UINT64_C(0x3956c25bf348b538), UINT64_C(0x59f111f1b605d019), UINT64_C(0x923f82a4af194f9b), UINT64_C(0xab1c5ed5da6d8118), + 0); + RTSHA512_EIGHT(UINT64_C(0xd807aa98a3030242), UINT64_C(0x12835b0145706fbe), UINT64_C(0x243185be4ee4b28c), UINT64_C(0x550c7dc3d5ffb4e2), + UINT64_C(0x72be5d74f27b896f), UINT64_C(0x80deb1fe3b1696b1), UINT64_C(0x9bdc06a725c71235), UINT64_C(0xc19bf174cf692694), + 8); + RTSHA512_EIGHT(UINT64_C(0xe49b69c19ef14ad2), UINT64_C(0xefbe4786384f25e3), UINT64_C(0x0fc19dc68b8cd5b5), UINT64_C(0x240ca1cc77ac9c65), + UINT64_C(0x2de92c6f592b0275), UINT64_C(0x4a7484aa6ea6e483), UINT64_C(0x5cb0a9dcbd41fbd4), UINT64_C(0x76f988da831153b5), + 16); + RTSHA512_EIGHT(UINT64_C(0x983e5152ee66dfab), UINT64_C(0xa831c66d2db43210), UINT64_C(0xb00327c898fb213f), UINT64_C(0xbf597fc7beef0ee4), + UINT64_C(0xc6e00bf33da88fc2), UINT64_C(0xd5a79147930aa725), UINT64_C(0x06ca6351e003826f), UINT64_C(0x142929670a0e6e70), + 24); + RTSHA512_EIGHT(UINT64_C(0x27b70a8546d22ffc), UINT64_C(0x2e1b21385c26c926), UINT64_C(0x4d2c6dfc5ac42aed), UINT64_C(0x53380d139d95b3df), + UINT64_C(0x650a73548baf63de), UINT64_C(0x766a0abb3c77b2a8), UINT64_C(0x81c2c92e47edaee6), UINT64_C(0x92722c851482353b), + 32); + RTSHA512_EIGHT(UINT64_C(0xa2bfe8a14cf10364), UINT64_C(0xa81a664bbc423001), UINT64_C(0xc24b8b70d0f89791), UINT64_C(0xc76c51a30654be30), + UINT64_C(0xd192e819d6ef5218), UINT64_C(0xd69906245565a910), UINT64_C(0xf40e35855771202a), UINT64_C(0x106aa07032bbd1b8), + 40); + RTSHA512_EIGHT(UINT64_C(0x19a4c116b8d2d0c8), UINT64_C(0x1e376c085141ab53), UINT64_C(0x2748774cdf8eeb99), UINT64_C(0x34b0bcb5e19b48a8), + UINT64_C(0x391c0cb3c5c95a63), UINT64_C(0x4ed8aa4ae3418acb), UINT64_C(0x5b9cca4f7763e373), UINT64_C(0x682e6ff3d6b2b8a3), + 48); + RTSHA512_EIGHT(UINT64_C(0x748f82ee5defb2fc), UINT64_C(0x78a5636f43172f60), UINT64_C(0x84c87814a1f0ab72), UINT64_C(0x8cc702081a6439ec), + UINT64_C(0x90befffa23631e28), UINT64_C(0xa4506cebde82bde9), UINT64_C(0xbef9a3f7b2c67915), UINT64_C(0xc67178f2e372532b), + 56); + RTSHA512_EIGHT(UINT64_C(0xca273eceea26619c), UINT64_C(0xd186b8c721c0c207), UINT64_C(0xeada7dd6cde0eb1e), UINT64_C(0xf57d4f7fee6ed178), + UINT64_C(0x06f067aa72176fba), UINT64_C(0x0a637dc5a2c898a6), UINT64_C(0x113f9804bef90dae), UINT64_C(0x1b710b35131c471b), + 64); + RTSHA512_EIGHT(UINT64_C(0x28db77f523047d84), UINT64_C(0x32caab7b40c72493), UINT64_C(0x3c9ebe0a15c9bebc), UINT64_C(0x431d67c49c100d4c), + UINT64_C(0x4cc5d4becb3e42b6), UINT64_C(0x597f299cfc657e2a), UINT64_C(0x5fcb6fab3ad6faec), UINT64_C(0x6c44198c4a475817), + 72); +#else for (unsigned iWord = 0; iWord < RT_ELEMENTS(pCtx->AltPrivate.auW); iWord++) { @@ -264,4 +421,5 @@ uA = uT1 + uT2; } +#endif pCtx->AltPrivate.auH[0] += uA; @@ -308,5 +466,5 @@ } - if (!((uintptr_t)pbBuf & 3)) + if (!((uintptr_t)pbBuf & 7)) { /*