[PATCH] staging: rtl8192u: Fix warnings about endianness

[Arnd Bergmann]

On Wed, Feb 15, 2017 at 8:54 AM, maomao xu <albert008.xu at gmail.com> wrote:
> drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c:177:16: warning: cast to restricted __le16
> drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c:177:16: warning: cast to restricted __le16
> drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c:177:16: warning: cast to restricted __le16
> drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c:177:16: warning: cast to restricted __le16
> drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c:177:16: warning: cast to restricted __le16
> drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c:177:16: warning: cast to restricted __le16
> drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c:177:16: warning: cast to restricted __le16
> drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c:177:16: warning: cast to restricted __le16
> drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c:177:16: warning: cast to restricted __le16
>
> Signed-off-by: maomao xu <albert008.xu at gmail.com>
>
> diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c
> index 2453413..fb171bd 100644
> --- a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c
> +++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c
> @@ -172,7 +172,7 @@ static inline u16 Mk16(u8 hi, u8 lo)
>  }
>
>
> -static inline u16 Mk16_le(u16 *v)
> +static inline u16 Mk16_le(__le16 *v)
>  {
>         return le16_to_cpu(*v);
>  }
> @@ -264,15 +264,15 @@ static void tkip_mixing_phase2(u8 *WEPSeed, const u8 *TK, const u16 *TTAK,
>         PPK[5] = TTAK[4] + IV16;
>
>         /* Step 2 - 96-bit bijective mixing using S-box */
> -       PPK[0] += _S_(PPK[5] ^ Mk16_le((u16 *) &TK[0]));
> -       PPK[1] += _S_(PPK[0] ^ Mk16_le((u16 *) &TK[2]));
> -       PPK[2] += _S_(PPK[1] ^ Mk16_le((u16 *) &TK[4]));
> -       PPK[3] += _S_(PPK[2] ^ Mk16_le((u16 *) &TK[6]));
> -       PPK[4] += _S_(PPK[3] ^ Mk16_le((u16 *) &TK[8]));
> -       PPK[5] += _S_(PPK[4] ^ Mk16_le((u16 *) &TK[10]));
> -
> -       PPK[0] += RotR1(PPK[5] ^ Mk16_le((u16 *) &TK[12]));
> -       PPK[1] += RotR1(PPK[0] ^ Mk16_le((u16 *) &TK[14]));
> +       PPK[0] += _S_(PPK[5] ^ Mk16_le((__le16 *) &TK[0]));
> +       PPK[1] += _S_(PPK[0] ^ Mk16_le((__le16 *) &TK[2]));
> +       PPK[2] += _S_(PPK[1] ^ Mk16_le((__le16 *) &TK[4]));
> +       PPK[3] += _S_(PPK[2] ^ Mk16_le((__le16 *) &TK[6]));
> +       PPK[4] += _S_(PPK[3] ^ Mk16_le((__le16 *) &TK[8]));
> +       PPK[5] += _S_(PPK[4] ^ Mk16_le((__le16 *) &TK[10]));
> +
> +       PPK[0] += RotR1(PPK[5] ^ Mk16_le((__le16 *) &TK[12]));
> +       PPK[1] += RotR1(PPK[0] ^ Mk16_le((__le16 *) &TK[14]));
>         PPK[2] += RotR1(PPK[1]);
>         PPK[3] += RotR1(PPK[2]);
>         PPK[4] += RotR1(PPK[3]);
> @@ -285,7 +285,7 @@ static void tkip_mixing_phase2(u8 *WEPSeed, const u8 *TK, const u16 *TTAK,
>         WEPSeed[0] = Hi8(IV16);
>         WEPSeed[1] = (Hi8(IV16) | 0x20) & 0x7F;
>         WEPSeed[2] = Lo8(IV16);
> -       WEPSeed[3] = Lo8((PPK[5] ^ Mk16_le((u16 *) &TK[0])) >> 1);
> +       WEPSeed[3] = Lo8((PPK[5] ^ Mk16_le((__le16 *) &TK[0])) >> 1);
>
>  #ifdef __BIG_ENDIAN
>         {

I see the same warning was addressed very differently in 99277c1f9962
("Staging: rtl8192e: Fix Sparse warning of cast to restricted __le16 in
 rtllib_crypt_tkip.c"), which was for a close relative of that driver.

Only one of the two approaches (at most) can be correct, so we
regardless of your patch either rtl8192e or rtl8192u is broken on
big-endian machines.

I suspect rtl8192e is broken, and your patch is relatively harmless
(we could debate whether it is the cleanest approach, but it doesn't
change behavior and it avoids the warning), but maybe Peter
Waskiewicz knows something about the tkip algorithm that I don't
see.

     Arnd