{-# LANGUAGE CPP #-}
{-# LANGUAGE LambdaCase #-}

#if __GLASGOW_HASKELL__ <= 808
-- GHC 8.10 deprecates this flag, but GHC 8.8 needs it
-- emitPrimOp is quite large
{-# OPTIONS_GHC -fmax-pmcheck-iterations=4000000 #-}
#endif

----------------------------------------------------------------------------
--
-- Stg to C--: primitive operations
--
-- (c) The University of Glasgow 2004-2006
--
-----------------------------------------------------------------------------

module GHC.StgToCmm.Prim (
   cgOpApp,
   cgPrimOp, -- internal(ish), used by cgCase to get code for a
             -- comparison without also turning it into a Bool.
   shouldInlinePrimOp
 ) where

#include "HsVersions.h"

import GhcPrelude hiding ((<*>))

import GHC.StgToCmm.Layout
import GHC.StgToCmm.Foreign
import GHC.StgToCmm.Env
import GHC.StgToCmm.Monad
import GHC.StgToCmm.Utils
import GHC.StgToCmm.Ticky
import GHC.StgToCmm.Heap
import GHC.StgToCmm.Prof ( costCentreFrom )

import DynFlags
import GHC.Platform
import BasicTypes
import BlockId
import MkGraph
import StgSyn
import Cmm
import Module   ( rtsUnitId )
import Type     ( Type, tyConAppTyCon )
import TyCon
import CLabel
import CmmUtils
import PrimOp
import SMRep
import FastString
import Outputable
import Util
import Data.Maybe

import Data.Bits ((.&.), bit)
import Control.Monad (liftM, when, unless)

------------------------------------------------------------------------
--      Primitive operations and foreign calls
------------------------------------------------------------------------

{- Note [Foreign call results]
   ~~~~~~~~~~~~~~~~~~~~~~~~~~~
A foreign call always returns an unboxed tuple of results, one
of which is the state token.  This seems to happen even for pure
calls.

Even if we returned a single result for pure calls, it'd still be
right to wrap it in a singleton unboxed tuple, because the result
might be a Haskell closure pointer, we don't want to evaluate it. -}

----------------------------------
cgOpApp :: StgOp        -- The op
        -> [StgArg]     -- Arguments
        -> Type         -- Result type (always an unboxed tuple)
        -> FCode ReturnKind

-- Foreign calls
cgOpApp :: StgOp -> [StgArg] -> Type -> FCode ReturnKind
cgOpApp (StgFCallOp ForeignCall
fcall Type
ty) [StgArg]
stg_args Type
res_ty
  = ForeignCall -> Type -> [StgArg] -> Type -> FCode ReturnKind
cgForeignCall ForeignCall
fcall Type
ty [StgArg]
stg_args Type
res_ty
      -- Note [Foreign call results]

-- tagToEnum# is special: we need to pull the constructor
-- out of the table, and perform an appropriate return.

cgOpApp (StgPrimOp PrimOp
TagToEnumOp) [StgArg
arg] Type
res_ty
  = ASSERT(isEnumerationTyCon tycon)
    do  { DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        ; [CmmExpr]
args' <- [StgArg] -> FCode [CmmExpr]
getNonVoidArgAmodes [StgArg
arg]
        ; let amode :: CmmExpr
amode = case [CmmExpr]
args' of [CmmExpr
amode] -> CmmExpr
amode
                                    [CmmExpr]
_ -> String -> CmmExpr
forall a. String -> a
panic String
"TagToEnumOp had void arg"
        ; [CmmExpr] -> FCode ReturnKind
emitReturn [DynFlags -> TyCon -> CmmExpr -> CmmExpr
tagToClosure DynFlags
dflags TyCon
tycon CmmExpr
amode] }
   where
          -- If you're reading this code in the attempt to figure
          -- out why the compiler panic'ed here, it is probably because
          -- you used tagToEnum# in a non-monomorphic setting, e.g.,
          --         intToTg :: Enum a => Int -> a ; intToTg (I# x#) = tagToEnum# x#
          -- That won't work.
        tycon :: TyCon
tycon = Type -> TyCon
tyConAppTyCon Type
res_ty

cgOpApp (StgPrimOp PrimOp
primop) [StgArg]
args Type
res_ty = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    [CmmExpr]
cmm_args <- [StgArg] -> FCode [CmmExpr]
getNonVoidArgAmodes [StgArg]
args
    case DynFlags -> PrimOp -> [CmmExpr] -> Maybe ([LocalReg] -> FCode ())
emitPrimOp DynFlags
dflags PrimOp
primop [CmmExpr]
cmm_args of
        Maybe ([LocalReg] -> FCode ())
Nothing -> do  -- out-of-line
          let fun :: CmmExpr
fun = CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel (PrimOp -> CLabel
mkRtsPrimOpLabel PrimOp
primop))
          (Convention, Convention)
-> CmmExpr -> [CmmExpr] -> FCode ReturnKind
emitCall (Convention
NativeNodeCall, Convention
NativeReturn) CmmExpr
fun [CmmExpr]
cmm_args

        Just [LocalReg] -> FCode ()
f  -- inline
          | ReturnsPrim PrimRep
VoidRep <- PrimOpResultInfo
result_info
          -> do [LocalReg] -> FCode ()
f []
                [CmmExpr] -> FCode ReturnKind
emitReturn []

          | ReturnsPrim PrimRep
rep <- PrimOpResultInfo
result_info
          -> do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
                LocalReg
res <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp (DynFlags -> PrimRep -> CmmType
primRepCmmType DynFlags
dflags PrimRep
rep)
                [LocalReg] -> FCode ()
f [LocalReg
res]
                [CmmExpr] -> FCode ReturnKind
emitReturn [CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
res)]

          | ReturnsAlg TyCon
tycon <- PrimOpResultInfo
result_info, TyCon -> Bool
isUnboxedTupleTyCon TyCon
tycon
          -> do ([LocalReg]
regs, [ForeignHint]
_hints) <- Type -> FCode ([LocalReg], [ForeignHint])
newUnboxedTupleRegs Type
res_ty
                [LocalReg] -> FCode ()
f [LocalReg]
regs
                [CmmExpr] -> FCode ReturnKind
emitReturn ((LocalReg -> CmmExpr) -> [LocalReg] -> [CmmExpr]
forall a b. (a -> b) -> [a] -> [b]
map (CmmReg -> CmmExpr
CmmReg (CmmReg -> CmmExpr) -> (LocalReg -> CmmReg) -> LocalReg -> CmmExpr
forall b c a. (b -> c) -> (a -> b) -> a -> c
. LocalReg -> CmmReg
CmmLocal) [LocalReg]
regs)

          | Bool
otherwise -> String -> FCode ReturnKind
forall a. String -> a
panic String
"cgPrimop"
          where
             result_info :: PrimOpResultInfo
result_info = PrimOp -> PrimOpResultInfo
getPrimOpResultInfo PrimOp
primop

cgOpApp (StgPrimCallOp PrimCall
primcall) [StgArg]
args Type
_res_ty
  = do  { [CmmExpr]
cmm_args <- [StgArg] -> FCode [CmmExpr]
getNonVoidArgAmodes [StgArg]
args
        ; let fun :: CmmExpr
fun = CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel (PrimCall -> CLabel
mkPrimCallLabel PrimCall
primcall))
        ; (Convention, Convention)
-> CmmExpr -> [CmmExpr] -> FCode ReturnKind
emitCall (Convention
NativeNodeCall, Convention
NativeReturn) CmmExpr
fun [CmmExpr]
cmm_args }

-- | Interpret the argument as an unsigned value, assuming the value
-- is given in two-complement form in the given width.
--
-- Example: @asUnsigned W64 (-1)@ is 18446744073709551615.
--
-- This function is used to work around the fact that many array
-- primops take Int# arguments, but we interpret them as unsigned
-- quantities in the code gen. This means that we have to be careful
-- every time we work on e.g. a CmmInt literal that corresponds to the
-- array size, as it might contain a negative Integer value if the
-- user passed a value larger than 2^(wORD_SIZE_IN_BITS-1) as the Int#
-- literal.
asUnsigned :: Width -> Integer -> Integer
asUnsigned :: Width -> Integer -> Integer
asUnsigned Width
w Integer
n = Integer
n Integer -> Integer -> Integer
forall a. Bits a => a -> a -> a
.&. (Int -> Integer
forall a. Bits a => Int -> a
bit (Width -> Int
widthInBits Width
w) Integer -> Integer -> Integer
forall a. Num a => a -> a -> a
- Integer
1)

---------------------------------------------------
cgPrimOp   :: [LocalReg]        -- where to put the results
           -> PrimOp            -- the op
           -> [StgArg]          -- arguments
           -> FCode ()

cgPrimOp :: [LocalReg] -> PrimOp -> [StgArg] -> FCode ()
cgPrimOp [LocalReg]
results PrimOp
op [StgArg]
args = do
  DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
  [CmmExpr]
arg_exprs <- [StgArg] -> FCode [CmmExpr]
getNonVoidArgAmodes [StgArg]
args
  case DynFlags -> PrimOp -> [CmmExpr] -> Maybe ([LocalReg] -> FCode ())
emitPrimOp DynFlags
dflags PrimOp
op [CmmExpr]
arg_exprs of
    Maybe ([LocalReg] -> FCode ())
Nothing -> String -> FCode ()
forall a. String -> a
panic String
"External prim op"
    Just [LocalReg] -> FCode ()
f -> [LocalReg] -> FCode ()
f [LocalReg]
results


------------------------------------------------------------------------
--      Emitting code for a primop
------------------------------------------------------------------------

shouldInlinePrimOp :: DynFlags -> PrimOp -> [CmmExpr] -> Bool
shouldInlinePrimOp :: DynFlags -> PrimOp -> [CmmExpr] -> Bool
shouldInlinePrimOp DynFlags
dflags PrimOp
op [CmmExpr]
args = Maybe ([LocalReg] -> FCode ()) -> Bool
forall a. Maybe a -> Bool
isJust (Maybe ([LocalReg] -> FCode ()) -> Bool)
-> Maybe ([LocalReg] -> FCode ()) -> Bool
forall a b. (a -> b) -> a -> b
$ DynFlags -> PrimOp -> [CmmExpr] -> Maybe ([LocalReg] -> FCode ())
emitPrimOp DynFlags
dflags PrimOp
op [CmmExpr]
args

-- TODO: Several primop implementations (e.g. 'doNewByteArrayOp') use
-- ByteOff (or some other fixed width signed type) to represent
-- array sizes or indices. This means that these will overflow for
-- large enough sizes.

-- TODO: Several primops, such as 'copyArray#', only have an inline
-- implementation (below) but could possibly have both an inline
-- implementation and an out-of-line implementation, just like
-- 'newArray#'. This would lower the amount of code generated,
-- hopefully without a performance impact (needs to be measured).

-- | The big function handling all the primops. The 'OpDest' function type
-- abstracts over a few common cases, and the "most manual" fallback.
--
-- In the simple case, there is just one implementation, and we emit that.
--
-- In more complex cases, there is a foreign call (out of line) fallback. This
-- might happen e.g. if there's enough static information, such as statically
-- know arguments.
dispatchPrimop
  :: DynFlags
  -> PrimOp            -- ^ The primop
  -> [CmmExpr]         -- ^ The primop arguments
  -> OpDest
dispatchPrimop :: DynFlags -> PrimOp -> [CmmExpr] -> OpDest
dispatchPrimop DynFlags
dflags = \case
  PrimOp
NewByteArrayOp_Char -> \case
    [(CmmLit (CmmInt Integer
n Width
w))]
      | Width -> Integer -> Integer
asUnsigned Width
w Integer
n Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone  (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [LocalReg
res] -> LocalReg -> Int -> FCode ()
doNewByteArrayOp LocalReg
res (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
NewArrayOp -> \case
    [(CmmLit (CmmInt Integer
n Width
w)), CmmExpr
init]
      | DynFlags -> Integer -> Integer
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags (Width -> Integer -> Integer
asUnsigned Width
w Integer
n) Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> LocalReg
-> SMRep
-> CLabel
-> [(CmmExpr, Int)]
-> Int
-> CmmExpr
-> FCode ()
doNewArrayOp LocalReg
res (DynFlags -> Int -> SMRep
arrPtrsRep DynFlags
dflags (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)) CLabel
mkMAP_DIRTY_infoLabel
        [ (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n),
           DynFlags -> Int
fixedHdrSize DynFlags
dflags Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DynFlags -> Int
oFFSET_StgMutArrPtrs_ptrs DynFlags
dflags)
        , (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (SMRep -> Int
nonHdrSizeW (DynFlags -> Int -> SMRep
arrPtrsRep DynFlags
dflags (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n))),
           DynFlags -> Int
fixedHdrSize DynFlags
dflags Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DynFlags -> Int
oFFSET_StgMutArrPtrs_size DynFlags
dflags)
        ]
        (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n) CmmExpr
init
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
CopyArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, CmmExpr
dst, CmmExpr
dst_off, (CmmLit (CmmInt Integer
n Width
_))] ->
      ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [] -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
doCopyArrayOp CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
dst_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
CopyMutableArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, CmmExpr
dst, CmmExpr
dst_off, (CmmLit (CmmInt Integer
n Width
_))] ->
      ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [] -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
doCopyMutableArrayOp CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
dst_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
CopyArrayArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, CmmExpr
dst, CmmExpr
dst_off, (CmmLit (CmmInt Integer
n Width
_))] ->
      ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [] -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
doCopyArrayOp CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
dst_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
CopyMutableArrayArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, CmmExpr
dst, CmmExpr
dst_off, (CmmLit (CmmInt Integer
n Width
_))] ->
      ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [] -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
doCopyMutableArrayOp CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
dst_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
CloneArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, (CmmLit (CmmInt Integer
n Width
w))]
      | DynFlags -> Integer -> Integer
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags (Width -> Integer -> Integer
asUnsigned Width
w Integer
n) Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [LocalReg
res] -> CLabel -> LocalReg -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCloneArray CLabel
mkMAP_FROZEN_CLEAN_infoLabel LocalReg
res CmmExpr
src CmmExpr
src_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
CloneMutableArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, (CmmLit (CmmInt Integer
n Width
w))]
      | DynFlags -> Integer -> Integer
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags (Width -> Integer -> Integer
asUnsigned Width
w Integer
n) Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [LocalReg
res] -> CLabel -> LocalReg -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCloneArray CLabel
mkMAP_DIRTY_infoLabel LocalReg
res CmmExpr
src CmmExpr
src_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
FreezeArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, (CmmLit (CmmInt Integer
n Width
w))]
      | DynFlags -> Integer -> Integer
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags (Width -> Integer -> Integer
asUnsigned Width
w Integer
n) Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [LocalReg
res] -> CLabel -> LocalReg -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCloneArray CLabel
mkMAP_FROZEN_CLEAN_infoLabel LocalReg
res CmmExpr
src CmmExpr
src_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
ThawArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, (CmmLit (CmmInt Integer
n Width
w))]
      | DynFlags -> Integer -> Integer
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags (Width -> Integer -> Integer
asUnsigned Width
w Integer
n) Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [LocalReg
res] -> CLabel -> LocalReg -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCloneArray CLabel
mkMAP_DIRTY_infoLabel LocalReg
res CmmExpr
src CmmExpr
src_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
NewSmallArrayOp -> \case
    [(CmmLit (CmmInt Integer
n Width
w)), CmmExpr
init]
      | DynFlags -> Integer -> Integer
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags (Width -> Integer -> Integer
asUnsigned Width
w Integer
n) Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [LocalReg
res] ->
        LocalReg
-> SMRep
-> CLabel
-> [(CmmExpr, Int)]
-> Int
-> CmmExpr
-> FCode ()
doNewArrayOp LocalReg
res (Int -> SMRep
smallArrPtrsRep (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)) CLabel
mkSMAP_DIRTY_infoLabel
        [ (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n),
           DynFlags -> Int
fixedHdrSize DynFlags
dflags Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DynFlags -> Int
oFFSET_StgSmallMutArrPtrs_ptrs DynFlags
dflags)
        ]
        (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n) CmmExpr
init
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
CopySmallArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, CmmExpr
dst, CmmExpr
dst_off, (CmmLit (CmmInt Integer
n Width
_))] ->
      ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [] -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
doCopySmallArrayOp CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
dst_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
CopySmallMutableArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, CmmExpr
dst, CmmExpr
dst_off, (CmmLit (CmmInt Integer
n Width
_))] ->
      ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [] -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
doCopySmallMutableArrayOp CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
dst_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
CloneSmallArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, (CmmLit (CmmInt Integer
n Width
w))]
      | DynFlags -> Integer -> Integer
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags (Width -> Integer -> Integer
asUnsigned Width
w Integer
n) Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [LocalReg
res] -> CLabel -> LocalReg -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCloneSmallArray CLabel
mkSMAP_FROZEN_CLEAN_infoLabel LocalReg
res CmmExpr
src CmmExpr
src_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
CloneSmallMutableArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, (CmmLit (CmmInt Integer
n Width
w))]
      | DynFlags -> Integer -> Integer
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags (Width -> Integer -> Integer
asUnsigned Width
w Integer
n) Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [LocalReg
res] -> CLabel -> LocalReg -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCloneSmallArray CLabel
mkSMAP_DIRTY_infoLabel LocalReg
res CmmExpr
src CmmExpr
src_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
FreezeSmallArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, (CmmLit (CmmInt Integer
n Width
w))]
      | DynFlags -> Integer -> Integer
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags (Width -> Integer -> Integer
asUnsigned Width
w Integer
n) Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [LocalReg
res] -> CLabel -> LocalReg -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCloneSmallArray CLabel
mkSMAP_FROZEN_CLEAN_infoLabel LocalReg
res CmmExpr
src CmmExpr
src_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

  PrimOp
ThawSmallArrayOp -> \case
    [CmmExpr
src, CmmExpr
src_off, (CmmLit (CmmInt Integer
n Width
w))]
      | DynFlags -> Integer -> Integer
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags (Width -> Integer -> Integer
asUnsigned Width
w Integer
n) Integer -> Integer -> Bool
forall a. Ord a => a -> a -> Bool
<= Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (DynFlags -> Int
maxInlineAllocSize DynFlags
dflags)
      -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \ [LocalReg
res] -> CLabel -> LocalReg -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCloneSmallArray CLabel
mkSMAP_DIRTY_infoLabel LocalReg
res CmmExpr
src CmmExpr
src_off (Integer -> Int
forall a. Num a => Integer -> a
fromInteger Integer
n)
    [CmmExpr]
_ -> OpDest
OpDest_External

-- First we handle various awkward cases specially.

  PrimOp
ParOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    -- for now, just implement this in a C function
    -- later, we might want to inline it.
    [(LocalReg, ForeignHint)]
-> CmmExpr -> [(CmmExpr, ForeignHint)] -> FCode ()
emitCCall
        [(LocalReg
res,ForeignHint
NoHint)]
        (CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel (FastString
-> Maybe Int -> ForeignLabelSource -> FunctionOrData -> CLabel
mkForeignLabel (String -> FastString
fsLit String
"newSpark") Maybe Int
forall a. Maybe a
Nothing ForeignLabelSource
ForeignLabelInExternalPackage FunctionOrData
IsFunction)))
        [(CmmExpr
baseExpr, ForeignHint
AddrHint), (CmmExpr
arg,ForeignHint
AddrHint)]

  PrimOp
SparkOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    -- returns the value of arg in res.  We're going to therefore
    -- refer to arg twice (once to pass to newSpark(), and once to
    -- assign to res), so put it in a temporary.
    LocalReg
tmp <- CmmExpr -> FCode LocalReg
assignTemp CmmExpr
arg
    LocalReg
tmp2 <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp (DynFlags -> CmmType
bWord DynFlags
dflags)
    [(LocalReg, ForeignHint)]
-> CmmExpr -> [(CmmExpr, ForeignHint)] -> FCode ()
emitCCall
        [(LocalReg
tmp2,ForeignHint
NoHint)]
        (CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel (FastString
-> Maybe Int -> ForeignLabelSource -> FunctionOrData -> CLabel
mkForeignLabel (String -> FastString
fsLit String
"newSpark") Maybe Int
forall a. Maybe a
Nothing ForeignLabelSource
ForeignLabelInExternalPackage FunctionOrData
IsFunction)))
        [(CmmExpr
baseExpr, ForeignHint
AddrHint), ((CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
tmp)), ForeignHint
AddrHint)]
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
tmp))

  PrimOp
GetCCSOfOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    let
      val :: CmmExpr
val
       | GeneralFlag -> DynFlags -> Bool
gopt GeneralFlag
Opt_SccProfilingOn DynFlags
dflags = DynFlags -> CmmExpr -> CmmExpr
costCentreFrom DynFlags
dflags (DynFlags -> CmmExpr -> CmmExpr
cmmUntag DynFlags
dflags CmmExpr
arg)
       | Bool
otherwise                      = CmmLit -> CmmExpr
CmmLit (DynFlags -> CmmLit
zeroCLit DynFlags
dflags)
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) CmmExpr
val

  PrimOp
GetCurrentCCSOp -> \[CmmExpr
_] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) CmmExpr
cccsExpr

  PrimOp
MyThreadIdOp -> \[] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) CmmExpr
currentTSOExpr

  PrimOp
ReadMutVarOp -> \[CmmExpr
mutv] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (DynFlags -> CmmExpr -> Int -> CmmType -> CmmExpr
cmmLoadIndexW DynFlags
dflags CmmExpr
mutv (DynFlags -> Int
fixedHdrSizeW DynFlags
dflags) (DynFlags -> CmmType
gcWord DynFlags
dflags))

  PrimOp
WriteMutVarOp -> \[CmmExpr
mutv, CmmExpr
var] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \res :: [LocalReg]
res@[] -> do
    CmmReg
old_val <- LocalReg -> CmmReg
CmmLocal (LocalReg -> CmmReg) -> FCode LocalReg -> FCode CmmReg
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp (DynFlags -> CmmExpr -> CmmType
cmmExprType DynFlags
dflags CmmExpr
var)
    CmmReg -> CmmExpr -> FCode ()
emitAssign CmmReg
old_val (DynFlags -> CmmExpr -> Int -> CmmType -> CmmExpr
cmmLoadIndexW DynFlags
dflags CmmExpr
mutv (DynFlags -> Int
fixedHdrSizeW DynFlags
dflags) (DynFlags -> CmmType
gcWord DynFlags
dflags))

    -- Without this write barrier, other CPUs may see this pointer before
    -- the writes for the closure it points to have occurred.
    -- Note that this also must come after we read the old value to ensure
    -- that the read of old_val comes before another core's write to the
    -- MutVar's value.
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall [LocalReg]
res CallishMachOp
MO_WriteBarrier []
    CmmExpr -> CmmExpr -> FCode ()
emitStore (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetW DynFlags
dflags CmmExpr
mutv (DynFlags -> Int
fixedHdrSizeW DynFlags
dflags)) CmmExpr
var
    [(LocalReg, ForeignHint)]
-> CmmExpr -> [(CmmExpr, ForeignHint)] -> FCode ()
emitCCall
            [{-no results-}]
            (CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel CLabel
mkDirty_MUT_VAR_Label))
            [(CmmExpr
baseExpr, ForeignHint
AddrHint), (CmmExpr
mutv, ForeignHint
AddrHint), (CmmReg -> CmmExpr
CmmReg CmmReg
old_val, ForeignHint
AddrHint)]

--  #define sizzeofByteArrayzh(r,a) \
--     r = ((StgArrBytes *)(a))->bytes
  PrimOp
SizeofByteArrayOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (DynFlags -> CmmExpr -> Int -> CmmType -> CmmExpr
cmmLoadIndexW DynFlags
dflags CmmExpr
arg (DynFlags -> Int
fixedHdrSizeW DynFlags
dflags) (DynFlags -> CmmType
bWord DynFlags
dflags))

--  #define sizzeofMutableByteArrayzh(r,a) \
--      r = ((StgArrBytes *)(a))->bytes
  PrimOp
SizeofMutableByteArrayOp -> DynFlags -> PrimOp -> [CmmExpr] -> OpDest
dispatchPrimop DynFlags
dflags PrimOp
SizeofByteArrayOp

--  #define getSizzeofMutableByteArrayzh(r,a) \
--      r = ((StgArrBytes *)(a))->bytes
  PrimOp
GetSizeofMutableByteArrayOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (DynFlags -> CmmExpr -> Int -> CmmType -> CmmExpr
cmmLoadIndexW DynFlags
dflags CmmExpr
arg (DynFlags -> Int
fixedHdrSizeW DynFlags
dflags) (DynFlags -> CmmType
bWord DynFlags
dflags))


--  #define touchzh(o)                  /* nothing */
  PrimOp
TouchOp -> \args :: [CmmExpr]
args@[CmmExpr
_] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \res :: [LocalReg]
res@[] -> do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall [LocalReg]
res CallishMachOp
MO_Touch [CmmExpr]
args

--  #define byteArrayContentszh(r,a) r = BYTE_ARR_CTS(a)
  PrimOp
ByteArrayContents_Char -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
arg (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags))

--  #define stableNameToIntzh(r,s)   (r = ((StgStableName *)s)->sn)
  PrimOp
StableNameToIntOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (DynFlags -> CmmExpr -> Int -> CmmType -> CmmExpr
cmmLoadIndexW DynFlags
dflags CmmExpr
arg (DynFlags -> Int
fixedHdrSizeW DynFlags
dflags) (DynFlags -> CmmType
bWord DynFlags
dflags))

  PrimOp
ReallyUnsafePtrEqualityOp -> \[CmmExpr
arg1, CmmExpr
arg2] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordEq DynFlags
dflags) [CmmExpr
arg1,CmmExpr
arg2])

--  #define addrToHValuezh(r,a) r=(P_)a
  PrimOp
AddrToAnyOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) CmmExpr
arg

--  #define hvalueToAddrzh(r, a) r=(W_)a
  PrimOp
AnyToAddrOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) CmmExpr
arg

{- Freezing arrays-of-ptrs requires changing an info table, for the
   benefit of the generational collector.  It needs to scavenge mutable
   objects, even if they are in old space.  When they become immutable,
   they can be removed from this scavenge list.  -}

--  #define unsafeFreezzeArrayzh(r,a)
--      {
--        SET_INFO((StgClosure *)a,&stg_MUT_ARR_PTRS_FROZEN_DIRTY_info);
--        r = a;
--      }
  PrimOp
UnsafeFreezeArrayOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ [CmmAGraph] -> CmmAGraph
catAGraphs
      [ CmmExpr -> CmmExpr -> CmmAGraph
setInfo CmmExpr
arg (CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel CLabel
mkMAP_FROZEN_DIRTY_infoLabel)),
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) CmmExpr
arg ]
  PrimOp
UnsafeFreezeArrayArrayOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ [CmmAGraph] -> CmmAGraph
catAGraphs
      [ CmmExpr -> CmmExpr -> CmmAGraph
setInfo CmmExpr
arg (CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel CLabel
mkMAP_FROZEN_DIRTY_infoLabel)),
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) CmmExpr
arg ]
  PrimOp
UnsafeFreezeSmallArrayOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ [CmmAGraph] -> CmmAGraph
catAGraphs
      [ CmmExpr -> CmmExpr -> CmmAGraph
setInfo CmmExpr
arg (CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel CLabel
mkSMAP_FROZEN_DIRTY_infoLabel)),
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) CmmExpr
arg ]

--  #define unsafeFreezzeByteArrayzh(r,a)       r=(a)
  PrimOp
UnsafeFreezeByteArrayOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) CmmExpr
arg

-- Reading/writing pointer arrays

  PrimOp
ReadArrayOp -> \[CmmExpr
obj, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadPtrArrayOp LocalReg
res CmmExpr
obj CmmExpr
ix
  PrimOp
IndexArrayOp -> \[CmmExpr
obj, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadPtrArrayOp LocalReg
res CmmExpr
obj CmmExpr
ix
  PrimOp
WriteArrayOp -> \[CmmExpr
obj, CmmExpr
ix, CmmExpr
v] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doWritePtrArrayOp CmmExpr
obj CmmExpr
ix CmmExpr
v

  PrimOp
IndexArrayArrayOp_ByteArray -> \[CmmExpr
obj, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadPtrArrayOp LocalReg
res CmmExpr
obj CmmExpr
ix
  PrimOp
IndexArrayArrayOp_ArrayArray -> \[CmmExpr
obj, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadPtrArrayOp LocalReg
res CmmExpr
obj CmmExpr
ix
  PrimOp
ReadArrayArrayOp_ByteArray -> \[CmmExpr
obj, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadPtrArrayOp LocalReg
res CmmExpr
obj CmmExpr
ix
  PrimOp
ReadArrayArrayOp_MutableByteArray -> \[CmmExpr
obj, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadPtrArrayOp LocalReg
res CmmExpr
obj CmmExpr
ix
  PrimOp
ReadArrayArrayOp_ArrayArray -> \[CmmExpr
obj, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadPtrArrayOp LocalReg
res CmmExpr
obj CmmExpr
ix
  PrimOp
ReadArrayArrayOp_MutableArrayArray -> \[CmmExpr
obj, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadPtrArrayOp LocalReg
res CmmExpr
obj CmmExpr
ix
  PrimOp
WriteArrayArrayOp_ByteArray -> \[CmmExpr
obj,CmmExpr
ix,CmmExpr
v] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doWritePtrArrayOp CmmExpr
obj CmmExpr
ix CmmExpr
v
  PrimOp
WriteArrayArrayOp_MutableByteArray -> \[CmmExpr
obj,CmmExpr
ix,CmmExpr
v] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doWritePtrArrayOp CmmExpr
obj CmmExpr
ix CmmExpr
v
  PrimOp
WriteArrayArrayOp_ArrayArray -> \[CmmExpr
obj,CmmExpr
ix,CmmExpr
v] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doWritePtrArrayOp CmmExpr
obj CmmExpr
ix CmmExpr
v
  PrimOp
WriteArrayArrayOp_MutableArrayArray -> \[CmmExpr
obj,CmmExpr
ix,CmmExpr
v] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doWritePtrArrayOp CmmExpr
obj CmmExpr
ix CmmExpr
v

  PrimOp
ReadSmallArrayOp -> \[CmmExpr
obj, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadSmallPtrArrayOp LocalReg
res CmmExpr
obj CmmExpr
ix
  PrimOp
IndexSmallArrayOp -> \[CmmExpr
obj, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadSmallPtrArrayOp LocalReg
res CmmExpr
obj CmmExpr
ix
  PrimOp
WriteSmallArrayOp -> \[CmmExpr
obj,CmmExpr
ix,CmmExpr
v] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doWriteSmallPtrArrayOp CmmExpr
obj CmmExpr
ix CmmExpr
v

-- Getting the size of pointer arrays

  PrimOp
SizeofArrayOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (DynFlags -> CmmExpr -> Int -> CmmType -> CmmExpr
cmmLoadIndexW DynFlags
dflags CmmExpr
arg
      (DynFlags -> Int
fixedHdrSizeW DynFlags
dflags Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DynFlags -> Int -> Int
bytesToWordsRoundUp DynFlags
dflags (DynFlags -> Int
oFFSET_StgMutArrPtrs_ptrs DynFlags
dflags))
        (DynFlags -> CmmType
bWord DynFlags
dflags))
  PrimOp
SizeofMutableArrayOp -> DynFlags -> PrimOp -> [CmmExpr] -> OpDest
dispatchPrimop DynFlags
dflags PrimOp
SizeofArrayOp
  PrimOp
SizeofArrayArrayOp -> DynFlags -> PrimOp -> [CmmExpr] -> OpDest
dispatchPrimop DynFlags
dflags PrimOp
SizeofArrayOp
  PrimOp
SizeofMutableArrayArrayOp -> DynFlags -> PrimOp -> [CmmExpr] -> OpDest
dispatchPrimop DynFlags
dflags PrimOp
SizeofArrayOp
  PrimOp
SizeofSmallArrayOp -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res)
     (DynFlags -> CmmExpr -> Int -> CmmType -> CmmExpr
cmmLoadIndexW DynFlags
dflags CmmExpr
arg
     (DynFlags -> Int
fixedHdrSizeW DynFlags
dflags Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DynFlags -> Int -> Int
bytesToWordsRoundUp DynFlags
dflags (DynFlags -> Int
oFFSET_StgSmallMutArrPtrs_ptrs DynFlags
dflags))
        (DynFlags -> CmmType
bWord DynFlags
dflags))

  PrimOp
SizeofSmallMutableArrayOp -> DynFlags -> PrimOp -> [CmmExpr] -> OpDest
dispatchPrimop DynFlags
dflags PrimOp
SizeofSmallArrayOp
  PrimOp
GetSizeofSmallMutableArrayOp -> DynFlags -> PrimOp -> [CmmExpr] -> OpDest
dispatchPrimop DynFlags
dflags PrimOp
SizeofSmallArrayOp

-- IndexXXXoffAddr

  PrimOp
IndexOffAddrOp_Char -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord DynFlags
dflags)) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_WideChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Int -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Word -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Addr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Float -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f32 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Double -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f64 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_StablePtr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Int8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_8ToWord DynFlags
dflags)) CmmType
b8  [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Int16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_16ToWord DynFlags
dflags)) CmmType
b16 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Int32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_32ToWord DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Int64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Word8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord DynFlags
dflags)) CmmType
b8  [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Word16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_16ToWord DynFlags
dflags)) CmmType
b16 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Word32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexOffAddrOp_Word64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 [LocalReg]
res [CmmExpr]
args

-- ReadXXXoffAddr, which are identical, for our purposes, to IndexXXXoffAddr.

  PrimOp
ReadOffAddrOp_Char -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord DynFlags
dflags)) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_WideChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Int -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Word -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Addr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Float -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f32 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Double -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f64 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_StablePtr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Int8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_8ToWord DynFlags
dflags)) CmmType
b8  [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Int16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_16ToWord DynFlags
dflags)) CmmType
b16 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Int32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_32ToWord DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Int64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Word8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord DynFlags
dflags)) CmmType
b8  [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Word16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_16ToWord DynFlags
dflags)) CmmType
b16 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Word32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadOffAddrOp_Word64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 [LocalReg]
res [CmmExpr]
args

-- IndexXXXArray

  PrimOp
IndexByteArrayOp_Char -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord DynFlags
dflags)) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_WideChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Int -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Addr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Float -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f32 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Double -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f64 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_StablePtr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Int8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_8ToWord DynFlags
dflags)) CmmType
b8  [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Int16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_16ToWord DynFlags
dflags)) CmmType
b16  [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Int32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_32ToWord DynFlags
dflags)) CmmType
b32  [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Int64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64  [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord DynFlags
dflags)) CmmType
b8  [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_16ToWord DynFlags
dflags)) CmmType
b16  [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32  [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64  [LocalReg]
res [CmmExpr]
args

-- ReadXXXArray, identical to IndexXXXArray.

  PrimOp
ReadByteArrayOp_Char -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord DynFlags
dflags)) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_WideChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Int -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Addr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Float -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f32 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Double -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f64 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_StablePtr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Int8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_8ToWord DynFlags
dflags)) CmmType
b8  [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Int16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_16ToWord DynFlags
dflags)) CmmType
b16  [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Int32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_32ToWord DynFlags
dflags)) CmmType
b32  [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Int64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64  [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord DynFlags
dflags)) CmmType
b8  [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_16ToWord DynFlags
dflags)) CmmType
b16  [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32  [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64  [LocalReg]
res [CmmExpr]
args

-- IndexWord8ArrayAsXXX

  PrimOp
IndexByteArrayOp_Word8AsChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord DynFlags
dflags)) CmmType
b8 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsWideChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsInt -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsWord -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsAddr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsFloat -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f32 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsDouble -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f64 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsStablePtr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsInt16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_16ToWord DynFlags
dflags)) CmmType
b16 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsInt32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_32ToWord DynFlags
dflags)) CmmType
b32 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsInt64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsWord16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_16ToWord DynFlags
dflags)) CmmType
b16 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsWord32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
IndexByteArrayOp_Word8AsWord64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 CmmType
b8 [LocalReg]
res [CmmExpr]
args

-- ReadInt8ArrayAsXXX, identical to IndexInt8ArrayAsXXX

  PrimOp
ReadByteArrayOp_Word8AsChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord DynFlags
dflags)) CmmType
b8 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsWideChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsInt -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsWord -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsAddr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsFloat -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f32 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsDouble -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing CmmType
f64 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsStablePtr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsInt16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_16ToWord DynFlags
dflags)) CmmType
b16 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsInt32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_32ToWord DynFlags
dflags)) CmmType
b32 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsInt64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsWord16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_16ToWord DynFlags
dflags)) CmmType
b16 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsWord32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)) CmmType
b32 CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
ReadByteArrayOp_Word8AsWord64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs   Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 CmmType
b8 [LocalReg]
res [CmmExpr]
args

-- WriteXXXoffAddr

  PrimOp
WriteOffAddrOp_Char -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo8 DynFlags
dflags))  CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_WideChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Int -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Word -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Addr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Float -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
f32 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Double -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
f64 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_StablePtr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Int8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo8 DynFlags
dflags))  CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Int16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo16 DynFlags
dflags)) CmmType
b16 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Int32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Int64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Word8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo8 DynFlags
dflags))  CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Word16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo16 DynFlags
dflags)) CmmType
b16 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Word32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteOffAddrOp_Word64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 [LocalReg]
res [CmmExpr]
args

-- WriteXXXArray

  PrimOp
WriteByteArrayOp_Char -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo8 DynFlags
dflags))  CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_WideChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Int -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Addr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Float -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
f32 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Double -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
f64 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_StablePtr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
bWord DynFlags
dflags) [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Int8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo8 DynFlags
dflags))  CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Int16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo16 DynFlags
dflags)) CmmType
b16 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Int32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Int64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo8 DynFlags
dflags))  CmmType
b8  [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo16 DynFlags
dflags)) CmmType
b16 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)) CmmType
b32 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b64 [LocalReg]
res [CmmExpr]
args

-- WriteInt8ArrayAsXXX

  PrimOp
WriteByteArrayOp_Word8AsChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo8 DynFlags
dflags))  CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsWideChar -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsInt -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsWord -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsAddr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsFloat -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsDouble -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsStablePtr -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsInt16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo16 DynFlags
dflags)) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsInt32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsInt64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsWord16 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo16 DynFlags
dflags)) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsWord32 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp (MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)) CmmType
b8 [LocalReg]
res [CmmExpr]
args
  PrimOp
WriteByteArrayOp_Word8AsWord64 -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
b8 [LocalReg]
res [CmmExpr]
args

-- Copying and setting byte arrays
  PrimOp
CopyByteArrayOp -> \[CmmExpr
src,CmmExpr
src_off,CmmExpr
dst,CmmExpr
dst_off,CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyByteArrayOp CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
dst_off CmmExpr
n
  PrimOp
CopyMutableByteArrayOp -> \[CmmExpr
src,CmmExpr
src_off,CmmExpr
dst,CmmExpr
dst_off,CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyMutableByteArrayOp CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
dst_off CmmExpr
n
  PrimOp
CopyByteArrayToAddrOp -> \[CmmExpr
src,CmmExpr
src_off,CmmExpr
dst,CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyByteArrayToAddrOp CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
n
  PrimOp
CopyMutableByteArrayToAddrOp -> \[CmmExpr
src,CmmExpr
src_off,CmmExpr
dst,CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyMutableByteArrayToAddrOp CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
n
  PrimOp
CopyAddrToByteArrayOp -> \[CmmExpr
src,CmmExpr
dst,CmmExpr
dst_off,CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyAddrToByteArrayOp CmmExpr
src CmmExpr
dst CmmExpr
dst_off CmmExpr
n
  PrimOp
SetByteArrayOp -> \[CmmExpr
ba,CmmExpr
off,CmmExpr
len,CmmExpr
c] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doSetByteArrayOp CmmExpr
ba CmmExpr
off CmmExpr
len CmmExpr
c

-- Comparing byte arrays
  PrimOp
CompareByteArraysOp -> \[CmmExpr
ba1,CmmExpr
ba1_off,CmmExpr
ba2,CmmExpr
ba2_off,CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCompareByteArraysOp LocalReg
res CmmExpr
ba1 CmmExpr
ba1_off CmmExpr
ba2 CmmExpr
ba2_off CmmExpr
n

  PrimOp
BSwap16Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitBSwapCall LocalReg
res CmmExpr
w Width
W16
  PrimOp
BSwap32Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitBSwapCall LocalReg
res CmmExpr
w Width
W32
  PrimOp
BSwap64Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitBSwapCall LocalReg
res CmmExpr
w Width
W64
  PrimOp
BSwapOp -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitBSwapCall LocalReg
res CmmExpr
w (DynFlags -> Width
wordWidth DynFlags
dflags)

  PrimOp
BRev8Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitBRevCall LocalReg
res CmmExpr
w Width
W8
  PrimOp
BRev16Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitBRevCall LocalReg
res CmmExpr
w Width
W16
  PrimOp
BRev32Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitBRevCall LocalReg
res CmmExpr
w Width
W32
  PrimOp
BRev64Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitBRevCall LocalReg
res CmmExpr
w Width
W64
  PrimOp
BRevOp -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitBRevCall LocalReg
res CmmExpr
w (DynFlags -> Width
wordWidth DynFlags
dflags)

-- Population count
  PrimOp
PopCnt8Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitPopCntCall LocalReg
res CmmExpr
w Width
W8
  PrimOp
PopCnt16Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitPopCntCall LocalReg
res CmmExpr
w Width
W16
  PrimOp
PopCnt32Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitPopCntCall LocalReg
res CmmExpr
w Width
W32
  PrimOp
PopCnt64Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitPopCntCall LocalReg
res CmmExpr
w Width
W64
  PrimOp
PopCntOp -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitPopCntCall LocalReg
res CmmExpr
w (DynFlags -> Width
wordWidth DynFlags
dflags)

-- Parallel bit deposit
  PrimOp
Pdep8Op -> \[CmmExpr
src, CmmExpr
mask] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPdepCall LocalReg
res CmmExpr
src CmmExpr
mask Width
W8
  PrimOp
Pdep16Op -> \[CmmExpr
src, CmmExpr
mask] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPdepCall LocalReg
res CmmExpr
src CmmExpr
mask Width
W16
  PrimOp
Pdep32Op -> \[CmmExpr
src, CmmExpr
mask] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPdepCall LocalReg
res CmmExpr
src CmmExpr
mask Width
W32
  PrimOp
Pdep64Op -> \[CmmExpr
src, CmmExpr
mask] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPdepCall LocalReg
res CmmExpr
src CmmExpr
mask Width
W64
  PrimOp
PdepOp -> \[CmmExpr
src, CmmExpr
mask] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPdepCall LocalReg
res CmmExpr
src CmmExpr
mask (DynFlags -> Width
wordWidth DynFlags
dflags)

-- Parallel bit extract
  PrimOp
Pext8Op -> \[CmmExpr
src, CmmExpr
mask] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPextCall LocalReg
res CmmExpr
src CmmExpr
mask Width
W8
  PrimOp
Pext16Op -> \[CmmExpr
src, CmmExpr
mask] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPextCall LocalReg
res CmmExpr
src CmmExpr
mask Width
W16
  PrimOp
Pext32Op -> \[CmmExpr
src, CmmExpr
mask] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPextCall LocalReg
res CmmExpr
src CmmExpr
mask Width
W32
  PrimOp
Pext64Op -> \[CmmExpr
src, CmmExpr
mask] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPextCall LocalReg
res CmmExpr
src CmmExpr
mask Width
W64
  PrimOp
PextOp -> \[CmmExpr
src, CmmExpr
mask] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPextCall LocalReg
res CmmExpr
src CmmExpr
mask (DynFlags -> Width
wordWidth DynFlags
dflags)

-- count leading zeros
  PrimOp
Clz8Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitClzCall LocalReg
res CmmExpr
w Width
W8
  PrimOp
Clz16Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitClzCall LocalReg
res CmmExpr
w Width
W16
  PrimOp
Clz32Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitClzCall LocalReg
res CmmExpr
w Width
W32
  PrimOp
Clz64Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitClzCall LocalReg
res CmmExpr
w Width
W64
  PrimOp
ClzOp -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitClzCall LocalReg
res CmmExpr
w (DynFlags -> Width
wordWidth DynFlags
dflags)

-- count trailing zeros
  PrimOp
Ctz8Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitCtzCall LocalReg
res CmmExpr
w Width
W8
  PrimOp
Ctz16Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitCtzCall LocalReg
res CmmExpr
w Width
W16
  PrimOp
Ctz32Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitCtzCall LocalReg
res CmmExpr
w Width
W32
  PrimOp
Ctz64Op -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitCtzCall LocalReg
res CmmExpr
w Width
W64
  PrimOp
CtzOp -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> Width -> FCode ()
emitCtzCall LocalReg
res CmmExpr
w (DynFlags -> Width
wordWidth DynFlags
dflags)

-- Unsigned int to floating point conversions
  PrimOp
Word2FloatOp -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall [LocalReg
res] (Width -> CallishMachOp
MO_UF_Conv Width
W32) [CmmExpr
w]
  PrimOp
Word2DoubleOp -> \[CmmExpr
w] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall [LocalReg
res] (Width -> CallishMachOp
MO_UF_Conv Width
W64) [CmmExpr
w]

-- SIMD primops
  (VecBroadcastOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr
e] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp
-> CmmType -> CmmExpr -> [CmmExpr] -> LocalReg -> FCode ()
doVecPackOp (DynFlags -> PrimOpVecCat -> Width -> Maybe MachOp
vecElemInjectCast DynFlags
dflags PrimOpVecCat
vcat Width
w) CmmType
ty CmmExpr
zeros (Int -> CmmExpr -> [CmmExpr]
forall a. Int -> a -> [a]
replicate Int
n CmmExpr
e) LocalReg
res
   where
    zeros :: CmmExpr
    zeros :: CmmExpr
zeros = CmmLit -> CmmExpr
CmmLit (CmmLit -> CmmExpr) -> CmmLit -> CmmExpr
forall a b. (a -> b) -> a -> b
$ [CmmLit] -> CmmLit
CmmVec (Int -> CmmLit -> [CmmLit]
forall a. Int -> a -> [a]
replicate Int
n CmmLit
zero)

    zero :: CmmLit
    zero :: CmmLit
zero = case PrimOpVecCat
vcat of
             PrimOpVecCat
IntVec   -> Integer -> Width -> CmmLit
CmmInt Integer
0 Width
w
             PrimOpVecCat
WordVec  -> Integer -> Width -> CmmLit
CmmInt Integer
0 Width
w
             PrimOpVecCat
FloatVec -> Rational -> Width -> CmmLit
CmmFloat Rational
0 Width
w

    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

  (VecPackOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
es -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Bool -> FCode () -> FCode ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when ([CmmExpr]
es [CmmExpr] -> Int -> Bool
forall a. [a] -> Int -> Bool
`lengthIsNot` Int
n) (FCode () -> FCode ()) -> FCode () -> FCode ()
forall a b. (a -> b) -> a -> b
$
        String -> FCode ()
forall a. String -> a
panic String
"emitPrimOp: VecPackOp has wrong number of arguments"
    Maybe MachOp
-> CmmType -> CmmExpr -> [CmmExpr] -> LocalReg -> FCode ()
doVecPackOp (DynFlags -> PrimOpVecCat -> Width -> Maybe MachOp
vecElemInjectCast DynFlags
dflags PrimOpVecCat
vcat Width
w) CmmType
ty CmmExpr
zeros [CmmExpr]
es LocalReg
res
   where
    zeros :: CmmExpr
    zeros :: CmmExpr
zeros = CmmLit -> CmmExpr
CmmLit (CmmLit -> CmmExpr) -> CmmLit -> CmmExpr
forall a b. (a -> b) -> a -> b
$ [CmmLit] -> CmmLit
CmmVec (Int -> CmmLit -> [CmmLit]
forall a. Int -> a -> [a]
replicate Int
n CmmLit
zero)

    zero :: CmmLit
    zero :: CmmLit
zero = case PrimOpVecCat
vcat of
             PrimOpVecCat
IntVec   -> Integer -> Width -> CmmLit
CmmInt Integer
0 Width
w
             PrimOpVecCat
WordVec  -> Integer -> Width -> CmmLit
CmmInt Integer
0 Width
w
             PrimOpVecCat
FloatVec -> Rational -> Width -> CmmLit
CmmFloat Rational
0 Width
w

    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

  (VecUnpackOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr
arg] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Bool -> FCode () -> FCode ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when ([LocalReg]
res [LocalReg] -> Int -> Bool
forall a. [a] -> Int -> Bool
`lengthIsNot` Int
n) (FCode () -> FCode ()) -> FCode () -> FCode ()
forall a b. (a -> b) -> a -> b
$
        String -> FCode ()
forall a. String -> a
panic String
"emitPrimOp: VecUnpackOp has wrong number of results"
    Maybe MachOp -> CmmType -> CmmExpr -> [LocalReg] -> FCode ()
doVecUnpackOp (DynFlags -> PrimOpVecCat -> Width -> Maybe MachOp
vecElemProjectCast DynFlags
dflags PrimOpVecCat
vcat Width
w) CmmType
ty CmmExpr
arg [LocalReg]
res
   where
    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

  (VecInsertOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr
v,CmmExpr
e,CmmExpr
i] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp
-> CmmType -> CmmExpr -> CmmExpr -> CmmExpr -> LocalReg -> FCode ()
doVecInsertOp (DynFlags -> PrimOpVecCat -> Width -> Maybe MachOp
vecElemInjectCast DynFlags
dflags PrimOpVecCat
vcat Width
w) CmmType
ty CmmExpr
v CmmExpr
e CmmExpr
i LocalReg
res
   where
    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

  (VecIndexByteArrayOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

  (VecReadByteArrayOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

  (VecWriteByteArrayOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

  (VecIndexOffAddrOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

  (VecReadOffAddrOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

  (VecWriteOffAddrOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

  (VecIndexScalarByteArrayOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs Maybe MachOp
forall a. Maybe a
Nothing CmmType
vecty CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    vecty :: CmmType
    vecty :: CmmType
vecty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Width -> CmmType
vecCmmCat PrimOpVecCat
vcat Width
w

  (VecReadScalarByteArrayOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs Maybe MachOp
forall a. Maybe a
Nothing CmmType
vecty CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    vecty :: CmmType
    vecty :: CmmType
vecty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Width -> CmmType
vecCmmCat PrimOpVecCat
vcat Width
w

  (VecWriteScalarByteArrayOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Width -> CmmType
vecCmmCat PrimOpVecCat
vcat Width
w

  (VecIndexScalarOffAddrOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOpAs Maybe MachOp
forall a. Maybe a
Nothing CmmType
vecty CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    vecty :: CmmType
    vecty :: CmmType
vecty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Width -> CmmType
vecCmmCat PrimOpVecCat
vcat Width
w

  (VecReadScalarOffAddrOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOpAs Maybe MachOp
forall a. Maybe a
Nothing CmmType
vecty CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    vecty :: CmmType
    vecty :: CmmType
vecty = PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
n Width
w

    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Width -> CmmType
vecCmmCat PrimOpVecCat
vcat Width
w

  (VecWriteScalarOffAddrOp PrimOpVecCat
vcat Int
n Width
w) -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> do
    DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
n Width
w
    Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
forall a. Maybe a
Nothing CmmType
ty [LocalReg]
res0 [CmmExpr]
args
   where
    ty :: CmmType
    ty :: CmmType
ty = PrimOpVecCat -> Width -> CmmType
vecCmmCat PrimOpVecCat
vcat Width
w

-- Prefetch
  PrimOp
PrefetchByteArrayOp3         -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchByteArrayOp Int
3  [CmmExpr]
args
  PrimOp
PrefetchMutableByteArrayOp3  -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchMutableByteArrayOp Int
3  [CmmExpr]
args
  PrimOp
PrefetchAddrOp3              -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchAddrOp  Int
3  [CmmExpr]
args
  PrimOp
PrefetchValueOp3             -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchValueOp Int
3 [CmmExpr]
args

  PrimOp
PrefetchByteArrayOp2         -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchByteArrayOp Int
2  [CmmExpr]
args
  PrimOp
PrefetchMutableByteArrayOp2  -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchMutableByteArrayOp Int
2  [CmmExpr]
args
  PrimOp
PrefetchAddrOp2              -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchAddrOp Int
2  [CmmExpr]
args
  PrimOp
PrefetchValueOp2             -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchValueOp Int
2 [CmmExpr]
args
  PrimOp
PrefetchByteArrayOp1         -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchByteArrayOp Int
1  [CmmExpr]
args
  PrimOp
PrefetchMutableByteArrayOp1  -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchMutableByteArrayOp Int
1  [CmmExpr]
args
  PrimOp
PrefetchAddrOp1              -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchAddrOp Int
1  [CmmExpr]
args
  PrimOp
PrefetchValueOp1             -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchValueOp Int
1 [CmmExpr]
args

  PrimOp
PrefetchByteArrayOp0         -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchByteArrayOp Int
0  [CmmExpr]
args
  PrimOp
PrefetchMutableByteArrayOp0  -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchMutableByteArrayOp Int
0  [CmmExpr]
args
  PrimOp
PrefetchAddrOp0              -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchAddrOp Int
0  [CmmExpr]
args
  PrimOp
PrefetchValueOp0             -> \[CmmExpr]
args -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    Int -> [CmmExpr] -> FCode ()
doPrefetchValueOp Int
0 [CmmExpr]
args

-- Atomic read-modify-write
  PrimOp
FetchAddByteArrayOp_Int -> \[CmmExpr
mba, CmmExpr
ix, CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg
-> AtomicMachOp
-> CmmExpr
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
doAtomicRMW LocalReg
res AtomicMachOp
AMO_Add CmmExpr
mba CmmExpr
ix (DynFlags -> CmmType
bWord DynFlags
dflags) CmmExpr
n
  PrimOp
FetchSubByteArrayOp_Int -> \[CmmExpr
mba, CmmExpr
ix, CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg
-> AtomicMachOp
-> CmmExpr
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
doAtomicRMW LocalReg
res AtomicMachOp
AMO_Sub CmmExpr
mba CmmExpr
ix (DynFlags -> CmmType
bWord DynFlags
dflags) CmmExpr
n
  PrimOp
FetchAndByteArrayOp_Int -> \[CmmExpr
mba, CmmExpr
ix, CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg
-> AtomicMachOp
-> CmmExpr
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
doAtomicRMW LocalReg
res AtomicMachOp
AMO_And CmmExpr
mba CmmExpr
ix (DynFlags -> CmmType
bWord DynFlags
dflags) CmmExpr
n
  PrimOp
FetchNandByteArrayOp_Int -> \[CmmExpr
mba, CmmExpr
ix, CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg
-> AtomicMachOp
-> CmmExpr
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
doAtomicRMW LocalReg
res AtomicMachOp
AMO_Nand CmmExpr
mba CmmExpr
ix (DynFlags -> CmmType
bWord DynFlags
dflags) CmmExpr
n
  PrimOp
FetchOrByteArrayOp_Int -> \[CmmExpr
mba, CmmExpr
ix, CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg
-> AtomicMachOp
-> CmmExpr
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
doAtomicRMW LocalReg
res AtomicMachOp
AMO_Or CmmExpr
mba CmmExpr
ix (DynFlags -> CmmType
bWord DynFlags
dflags) CmmExpr
n
  PrimOp
FetchXorByteArrayOp_Int -> \[CmmExpr
mba, CmmExpr
ix, CmmExpr
n] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg
-> AtomicMachOp
-> CmmExpr
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
doAtomicRMW LocalReg
res AtomicMachOp
AMO_Xor CmmExpr
mba CmmExpr
ix (DynFlags -> CmmType
bWord DynFlags
dflags) CmmExpr
n
  PrimOp
AtomicReadByteArrayOp_Int -> \[CmmExpr
mba, CmmExpr
ix] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg -> CmmExpr -> CmmExpr -> CmmType -> FCode ()
doAtomicReadByteArray LocalReg
res CmmExpr
mba CmmExpr
ix (DynFlags -> CmmType
bWord DynFlags
dflags)
  PrimOp
AtomicWriteByteArrayOp_Int -> \[CmmExpr
mba, CmmExpr
ix, CmmExpr
val] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[] -> do
    CmmExpr -> CmmExpr -> CmmType -> CmmExpr -> FCode ()
doAtomicWriteByteArray CmmExpr
mba CmmExpr
ix (DynFlags -> CmmType
bWord DynFlags
dflags) CmmExpr
val
  PrimOp
CasByteArrayOp_Int -> \[CmmExpr
mba, CmmExpr
ix, CmmExpr
old, CmmExpr
new] -> ([LocalReg] -> FCode ()) -> OpDest
OpDest_AllDone (([LocalReg] -> FCode ()) -> OpDest)
-> ([LocalReg] -> FCode ()) -> OpDest
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    LocalReg
-> CmmExpr -> CmmExpr -> CmmType -> CmmExpr -> CmmExpr -> FCode ()
doCasByteArray LocalReg
res CmmExpr
mba CmmExpr
ix (DynFlags -> CmmType
bWord DynFlags
dflags) CmmExpr
old CmmExpr
new

-- The rest just translate straightforwardly

  PrimOp
Int2WordOp      -> \[CmmExpr]
_ -> OpDest
OpDest_Nop
  PrimOp
Word2IntOp      -> \[CmmExpr]
_ -> OpDest
OpDest_Nop
  PrimOp
Int2AddrOp      -> \[CmmExpr]
_ -> OpDest
OpDest_Nop
  PrimOp
Addr2IntOp      -> \[CmmExpr]
_ -> OpDest
OpDest_Nop
  PrimOp
ChrOp           -> \[CmmExpr]
_ -> OpDest
OpDest_Nop  -- Int# and Char# are rep'd the same
  PrimOp
OrdOp           -> \[CmmExpr]
_ -> OpDest
OpDest_Nop

  PrimOp
Narrow8IntOp   -> \[CmmExpr]
_ -> (Width -> Width -> MachOp, Width) -> OpDest
OpDest_Narrow (Width -> Width -> MachOp
MO_SS_Conv, Width
W8)
  PrimOp
Narrow16IntOp  -> \[CmmExpr]
_ -> (Width -> Width -> MachOp, Width) -> OpDest
OpDest_Narrow (Width -> Width -> MachOp
MO_SS_Conv, Width
W16)
  PrimOp
Narrow32IntOp  -> \[CmmExpr]
_ -> (Width -> Width -> MachOp, Width) -> OpDest
OpDest_Narrow (Width -> Width -> MachOp
MO_SS_Conv, Width
W32)
  PrimOp
Narrow8WordOp  -> \[CmmExpr]
_ -> (Width -> Width -> MachOp, Width) -> OpDest
OpDest_Narrow (Width -> Width -> MachOp
MO_UU_Conv, Width
W8)
  PrimOp
Narrow16WordOp -> \[CmmExpr]
_ -> (Width -> Width -> MachOp, Width) -> OpDest
OpDest_Narrow (Width -> Width -> MachOp
MO_UU_Conv, Width
W16)
  PrimOp
Narrow32WordOp -> \[CmmExpr]
_ -> (Width -> Width -> MachOp, Width) -> OpDest
OpDest_Narrow (Width -> Width -> MachOp
MO_UU_Conv, Width
W32)

  PrimOp
DoublePowerOp  -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Pwr
  PrimOp
DoubleSinOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Sin
  PrimOp
DoubleCosOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Cos
  PrimOp
DoubleTanOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Tan
  PrimOp
DoubleSinhOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Sinh
  PrimOp
DoubleCoshOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Cosh
  PrimOp
DoubleTanhOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Tanh
  PrimOp
DoubleAsinOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Asin
  PrimOp
DoubleAcosOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Acos
  PrimOp
DoubleAtanOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Atan
  PrimOp
DoubleAsinhOp  -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Asinh
  PrimOp
DoubleAcoshOp  -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Acosh
  PrimOp
DoubleAtanhOp  -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Atanh
  PrimOp
DoubleLogOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Log
  PrimOp
DoubleLog1POp  -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Log1P
  PrimOp
DoubleExpOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Exp
  PrimOp
DoubleExpM1Op  -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_ExpM1
  PrimOp
DoubleSqrtOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F64_Sqrt

  PrimOp
FloatPowerOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Pwr
  PrimOp
FloatSinOp     -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Sin
  PrimOp
FloatCosOp     -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Cos
  PrimOp
FloatTanOp     -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Tan
  PrimOp
FloatSinhOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Sinh
  PrimOp
FloatCoshOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Cosh
  PrimOp
FloatTanhOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Tanh
  PrimOp
FloatAsinOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Asin
  PrimOp
FloatAcosOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Acos
  PrimOp
FloatAtanOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Atan
  PrimOp
FloatAsinhOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Asinh
  PrimOp
FloatAcoshOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Acosh
  PrimOp
FloatAtanhOp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Atanh
  PrimOp
FloatLogOp     -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Log
  PrimOp
FloatLog1POp   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Log1P
  PrimOp
FloatExpOp     -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Exp
  PrimOp
FloatExpM1Op   -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_ExpM1
  PrimOp
FloatSqrtOp    -> \[CmmExpr]
_ -> CallishMachOp -> OpDest
OpDest_Callish CallishMachOp
MO_F32_Sqrt

-- Native word signless ops

  PrimOp
IntAddOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordAdd DynFlags
dflags)
  PrimOp
IntSubOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSub DynFlags
dflags)
  PrimOp
WordAddOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordAdd DynFlags
dflags)
  PrimOp
WordSubOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSub DynFlags
dflags)
  PrimOp
AddrAddOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordAdd DynFlags
dflags)
  PrimOp
AddrSubOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSub DynFlags
dflags)

  PrimOp
IntEqOp        -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
  PrimOp
IntNeOp        -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordNe DynFlags
dflags)
  PrimOp
WordEqOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
  PrimOp
WordNeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordNe DynFlags
dflags)
  PrimOp
AddrEqOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
  PrimOp
AddrNeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordNe DynFlags
dflags)

  PrimOp
AndOp          -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordAnd DynFlags
dflags)
  PrimOp
OrOp           -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordOr DynFlags
dflags)
  PrimOp
XorOp          -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordXor DynFlags
dflags)
  PrimOp
NotOp          -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordNot DynFlags
dflags)
  PrimOp
SllOp          -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordShl DynFlags
dflags)
  PrimOp
SrlOp          -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordUShr DynFlags
dflags)

  PrimOp
AddrRemOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordURem DynFlags
dflags)

-- Native word signed ops

  PrimOp
IntMulOp        -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordMul DynFlags
dflags)
  PrimOp
IntMulMayOfloOp -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_MulMayOflo (DynFlags -> Width
wordWidth DynFlags
dflags))
  PrimOp
IntQuotOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSQuot DynFlags
dflags)
  PrimOp
IntRemOp        -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSRem DynFlags
dflags)
  PrimOp
IntNegOp        -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSNeg DynFlags
dflags)

  PrimOp
IntGeOp        -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSGe DynFlags
dflags)
  PrimOp
IntLeOp        -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSLe DynFlags
dflags)
  PrimOp
IntGtOp        -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSGt DynFlags
dflags)
  PrimOp
IntLtOp        -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSLt DynFlags
dflags)

  PrimOp
AndIOp         -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordAnd DynFlags
dflags)
  PrimOp
OrIOp          -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordOr DynFlags
dflags)
  PrimOp
XorIOp         -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordXor DynFlags
dflags)
  PrimOp
NotIOp         -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordNot DynFlags
dflags)
  PrimOp
ISllOp         -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordShl DynFlags
dflags)
  PrimOp
ISraOp         -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordSShr DynFlags
dflags)
  PrimOp
ISrlOp         -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordUShr DynFlags
dflags)

-- Native word unsigned ops

  PrimOp
WordGeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordUGe DynFlags
dflags)
  PrimOp
WordLeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordULe DynFlags
dflags)
  PrimOp
WordGtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordUGt DynFlags
dflags)
  PrimOp
WordLtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordULt DynFlags
dflags)

  PrimOp
WordMulOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordMul DynFlags
dflags)
  PrimOp
WordQuotOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordUQuot DynFlags
dflags)
  PrimOp
WordRemOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordURem DynFlags
dflags)

  PrimOp
AddrGeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordUGe DynFlags
dflags)
  PrimOp
AddrLeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordULe DynFlags
dflags)
  PrimOp
AddrGtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordUGt DynFlags
dflags)
  PrimOp
AddrLtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordULt DynFlags
dflags)

-- Int8# signed ops

  PrimOp
Int8Extend     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_SS_Conv Width
W8 (DynFlags -> Width
wordWidth DynFlags
dflags))
  PrimOp
Int8Narrow     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_SS_Conv (DynFlags -> Width
wordWidth DynFlags
dflags) Width
W8)
  PrimOp
Int8NegOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Neg Width
W8)
  PrimOp
Int8AddOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Add Width
W8)
  PrimOp
Int8SubOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Sub Width
W8)
  PrimOp
Int8MulOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Mul Width
W8)
  PrimOp
Int8QuotOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Quot Width
W8)
  PrimOp
Int8RemOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Rem Width
W8)

  PrimOp
Int8EqOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Eq Width
W8)
  PrimOp
Int8GeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Ge Width
W8)
  PrimOp
Int8GtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Gt Width
W8)
  PrimOp
Int8LeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Le Width
W8)
  PrimOp
Int8LtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Lt Width
W8)
  PrimOp
Int8NeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Ne Width
W8)

-- Word8# unsigned ops

  PrimOp
Word8Extend     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_UU_Conv Width
W8 (DynFlags -> Width
wordWidth DynFlags
dflags))
  PrimOp
Word8Narrow     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_UU_Conv (DynFlags -> Width
wordWidth DynFlags
dflags) Width
W8)
  PrimOp
Word8NotOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Not Width
W8)
  PrimOp
Word8AddOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Add Width
W8)
  PrimOp
Word8SubOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Sub Width
W8)
  PrimOp
Word8MulOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Mul Width
W8)
  PrimOp
Word8QuotOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Quot Width
W8)
  PrimOp
Word8RemOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Rem Width
W8)

  PrimOp
Word8EqOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Eq Width
W8)
  PrimOp
Word8GeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Ge Width
W8)
  PrimOp
Word8GtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Gt Width
W8)
  PrimOp
Word8LeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Le Width
W8)
  PrimOp
Word8LtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Lt Width
W8)
  PrimOp
Word8NeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Ne Width
W8)

-- Int16# signed ops

  PrimOp
Int16Extend     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_SS_Conv Width
W16 (DynFlags -> Width
wordWidth DynFlags
dflags))
  PrimOp
Int16Narrow     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_SS_Conv (DynFlags -> Width
wordWidth DynFlags
dflags) Width
W16)
  PrimOp
Int16NegOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Neg Width
W16)
  PrimOp
Int16AddOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Add Width
W16)
  PrimOp
Int16SubOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Sub Width
W16)
  PrimOp
Int16MulOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Mul Width
W16)
  PrimOp
Int16QuotOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Quot Width
W16)
  PrimOp
Int16RemOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Rem Width
W16)

  PrimOp
Int16EqOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Eq Width
W16)
  PrimOp
Int16GeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Ge Width
W16)
  PrimOp
Int16GtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Gt Width
W16)
  PrimOp
Int16LeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Le Width
W16)
  PrimOp
Int16LtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_S_Lt Width
W16)
  PrimOp
Int16NeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Ne Width
W16)

-- Word16# unsigned ops

  PrimOp
Word16Extend     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_UU_Conv Width
W16 (DynFlags -> Width
wordWidth DynFlags
dflags))
  PrimOp
Word16Narrow     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_UU_Conv (DynFlags -> Width
wordWidth DynFlags
dflags) Width
W16)
  PrimOp
Word16NotOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Not Width
W16)
  PrimOp
Word16AddOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Add Width
W16)
  PrimOp
Word16SubOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Sub Width
W16)
  PrimOp
Word16MulOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Mul Width
W16)
  PrimOp
Word16QuotOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Quot Width
W16)
  PrimOp
Word16RemOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Rem Width
W16)

  PrimOp
Word16EqOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Eq Width
W16)
  PrimOp
Word16GeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Ge Width
W16)
  PrimOp
Word16GtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Gt Width
W16)
  PrimOp
Word16LeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Le Width
W16)
  PrimOp
Word16LtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Lt Width
W16)
  PrimOp
Word16NeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Ne Width
W16)

-- Char# ops

  PrimOp
CharEqOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Eq (DynFlags -> Width
wordWidth DynFlags
dflags))
  PrimOp
CharNeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_Ne (DynFlags -> Width
wordWidth DynFlags
dflags))
  PrimOp
CharGeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Ge (DynFlags -> Width
wordWidth DynFlags
dflags))
  PrimOp
CharLeOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Le (DynFlags -> Width
wordWidth DynFlags
dflags))
  PrimOp
CharGtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Gt (DynFlags -> Width
wordWidth DynFlags
dflags))
  PrimOp
CharLtOp       -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_U_Lt (DynFlags -> Width
wordWidth DynFlags
dflags))

-- Double ops

  PrimOp
DoubleEqOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Eq Width
W64)
  PrimOp
DoubleNeOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Ne Width
W64)
  PrimOp
DoubleGeOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Ge Width
W64)
  PrimOp
DoubleLeOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Le Width
W64)
  PrimOp
DoubleGtOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Gt Width
W64)
  PrimOp
DoubleLtOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Lt Width
W64)

  PrimOp
DoubleAddOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Add Width
W64)
  PrimOp
DoubleSubOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Sub Width
W64)
  PrimOp
DoubleMulOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Mul Width
W64)
  PrimOp
DoubleDivOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Quot Width
W64)
  PrimOp
DoubleNegOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Neg Width
W64)

-- Float ops

  PrimOp
FloatEqOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Eq Width
W32)
  PrimOp
FloatNeOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Ne Width
W32)
  PrimOp
FloatGeOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Ge Width
W32)
  PrimOp
FloatLeOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Le Width
W32)
  PrimOp
FloatGtOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Gt Width
W32)
  PrimOp
FloatLtOp     -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Lt Width
W32)

  PrimOp
FloatAddOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Add  Width
W32)
  PrimOp
FloatSubOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Sub  Width
W32)
  PrimOp
FloatMulOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Mul  Width
W32)
  PrimOp
FloatDivOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Quot Width
W32)
  PrimOp
FloatNegOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> MachOp
MO_F_Neg  Width
W32)

-- Vector ops

  (VecAddOp  PrimOpVecCat
FloatVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_VF_Add  Int
n Width
w)
  (VecSubOp  PrimOpVecCat
FloatVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_VF_Sub  Int
n Width
w)
  (VecMulOp  PrimOpVecCat
FloatVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_VF_Mul  Int
n Width
w)
  (VecDivOp  PrimOpVecCat
FloatVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_VF_Quot Int
n Width
w)
  (VecQuotOp PrimOpVecCat
FloatVec Int
_ Width
_) -> \[CmmExpr]
_ -> String -> OpDest
forall a. String -> a
panic String
"unsupported primop"
  (VecRemOp  PrimOpVecCat
FloatVec Int
_ Width
_) -> \[CmmExpr]
_ -> String -> OpDest
forall a. String -> a
panic String
"unsupported primop"
  (VecNegOp  PrimOpVecCat
FloatVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_VF_Neg  Int
n Width
w)

  (VecAddOp  PrimOpVecCat
IntVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_V_Add   Int
n Width
w)
  (VecSubOp  PrimOpVecCat
IntVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_V_Sub   Int
n Width
w)
  (VecMulOp  PrimOpVecCat
IntVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_V_Mul   Int
n Width
w)
  (VecDivOp  PrimOpVecCat
IntVec Int
_ Width
_) -> \[CmmExpr]
_ -> String -> OpDest
forall a. String -> a
panic String
"unsupported primop"
  (VecQuotOp PrimOpVecCat
IntVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_VS_Quot Int
n Width
w)
  (VecRemOp  PrimOpVecCat
IntVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_VS_Rem  Int
n Width
w)
  (VecNegOp  PrimOpVecCat
IntVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_VS_Neg  Int
n Width
w)

  (VecAddOp  PrimOpVecCat
WordVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_V_Add   Int
n Width
w)
  (VecSubOp  PrimOpVecCat
WordVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_V_Sub   Int
n Width
w)
  (VecMulOp  PrimOpVecCat
WordVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_V_Mul   Int
n Width
w)
  (VecDivOp  PrimOpVecCat
WordVec Int
_ Width
_) -> \[CmmExpr]
_ -> String -> OpDest
forall a. String -> a
panic String
"unsupported primop"
  (VecQuotOp PrimOpVecCat
WordVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_VU_Quot Int
n Width
w)
  (VecRemOp  PrimOpVecCat
WordVec Int
n Width
w) -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Int -> Width -> MachOp
MO_VU_Rem  Int
n Width
w)
  (VecNegOp  PrimOpVecCat
WordVec Int
_ Width
_) -> \[CmmExpr]
_ -> String -> OpDest
forall a. String -> a
panic String
"unsupported primop"

-- Conversions

  PrimOp
Int2DoubleOp   -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_SF_Conv (DynFlags -> Width
wordWidth DynFlags
dflags) Width
W64)
  PrimOp
Double2IntOp   -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_FS_Conv Width
W64 (DynFlags -> Width
wordWidth DynFlags
dflags))

  PrimOp
Int2FloatOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_SF_Conv (DynFlags -> Width
wordWidth DynFlags
dflags) Width
W32)
  PrimOp
Float2IntOp    -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_FS_Conv Width
W32 (DynFlags -> Width
wordWidth DynFlags
dflags))

  PrimOp
Float2DoubleOp -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_FF_Conv Width
W32 Width
W64)
  PrimOp
Double2FloatOp -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (Width -> Width -> MachOp
MO_FF_Conv Width
W64 Width
W32)

-- Word comparisons masquerading as more exotic things.

  PrimOp
SameMutVarOp            -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
  PrimOp
SameMVarOp              -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
  PrimOp
SameMutableArrayOp      -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
  PrimOp
SameMutableByteArrayOp  -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
  PrimOp
SameMutableArrayArrayOp -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
  PrimOp
SameSmallMutableArrayOp -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
  PrimOp
SameTVarOp              -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
  PrimOp
EqStablePtrOp           -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)
-- See Note [Comparing stable names]
  PrimOp
EqStableNameOp          -> \[CmmExpr]
_ -> MachOp -> OpDest
OpDest_Translate (DynFlags -> MachOp
mo_wordEq DynFlags
dflags)

  PrimOp
IntQuotRemOp -> \[CmmExpr]
args -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc) Bool -> Bool -> Bool
&& Bool -> Bool
not ([CmmExpr] -> Bool
quotRemCanBeOptimized [CmmExpr]
args)
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_S_QuotRem  (DynFlags -> Width
wordWidth DynFlags
dflags))
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right (Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericIntQuotRemOp (DynFlags -> Width
wordWidth DynFlags
dflags))

  PrimOp
Int8QuotRemOp -> \[CmmExpr]
args -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc) Bool -> Bool -> Bool
&& Bool -> Bool
not ([CmmExpr] -> Bool
quotRemCanBeOptimized [CmmExpr]
args)
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_S_QuotRem Width
W8)
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right (Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericIntQuotRemOp Width
W8)

  PrimOp
Int16QuotRemOp -> \[CmmExpr]
args -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc) Bool -> Bool -> Bool
&& Bool -> Bool
not ([CmmExpr] -> Bool
quotRemCanBeOptimized [CmmExpr]
args)
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_S_QuotRem Width
W16)
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right (Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericIntQuotRemOp Width
W16)

  PrimOp
WordQuotRemOp -> \[CmmExpr]
args -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc) Bool -> Bool -> Bool
&& Bool -> Bool
not ([CmmExpr] -> Bool
quotRemCanBeOptimized [CmmExpr]
args)
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_U_QuotRem  (DynFlags -> Width
wordWidth DynFlags
dflags))
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right (Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericWordQuotRemOp (DynFlags -> Width
wordWidth DynFlags
dflags))

  PrimOp
WordQuotRem2Op -> \[CmmExpr]
_ -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if (Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc)) Bool -> Bool -> Bool
|| Bool
llvm
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_U_QuotRem2 (DynFlags -> Width
wordWidth DynFlags
dflags))
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right (DynFlags -> [LocalReg] -> [CmmExpr] -> FCode ()
genericWordQuotRem2Op DynFlags
dflags)

  PrimOp
Word8QuotRemOp -> \[CmmExpr]
args -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc) Bool -> Bool -> Bool
&& Bool -> Bool
not ([CmmExpr] -> Bool
quotRemCanBeOptimized [CmmExpr]
args)
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_U_QuotRem Width
W8)
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right (Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericWordQuotRemOp Width
W8)

  PrimOp
Word16QuotRemOp -> \[CmmExpr]
args -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc) Bool -> Bool -> Bool
&& Bool -> Bool
not ([CmmExpr] -> Bool
quotRemCanBeOptimized [CmmExpr]
args)
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_U_QuotRem Width
W16)
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right (Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericWordQuotRemOp Width
W16)

  PrimOp
WordAdd2Op -> \[CmmExpr]
_ -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if (Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc)) Bool -> Bool -> Bool
|| Bool
llvm
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_Add2       (DynFlags -> Width
wordWidth DynFlags
dflags))
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right [LocalReg] -> [CmmExpr] -> FCode ()
genericWordAdd2Op

  PrimOp
WordAddCOp -> \[CmmExpr]
_ -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if (Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc)) Bool -> Bool -> Bool
|| Bool
llvm
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_AddWordC   (DynFlags -> Width
wordWidth DynFlags
dflags))
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right [LocalReg] -> [CmmExpr] -> FCode ()
genericWordAddCOp

  PrimOp
WordSubCOp -> \[CmmExpr]
_ -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if (Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc)) Bool -> Bool -> Bool
|| Bool
llvm
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_SubWordC   (DynFlags -> Width
wordWidth DynFlags
dflags))
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right [LocalReg] -> [CmmExpr] -> FCode ()
genericWordSubCOp

  PrimOp
IntAddCOp -> \[CmmExpr]
_ -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if (Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc)) Bool -> Bool -> Bool
|| Bool
llvm
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_AddIntC    (DynFlags -> Width
wordWidth DynFlags
dflags))
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right [LocalReg] -> [CmmExpr] -> FCode ()
genericIntAddCOp

  PrimOp
IntSubCOp -> \[CmmExpr]
_ -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if (Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc)) Bool -> Bool -> Bool
|| Bool
llvm
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_SubIntC    (DynFlags -> Width
wordWidth DynFlags
dflags))
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right [LocalReg] -> [CmmExpr] -> FCode ()
genericIntSubCOp

  PrimOp
WordMul2Op -> \[CmmExpr]
_ -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if Bool
ncg Bool -> Bool -> Bool
&& (Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc) Bool -> Bool -> Bool
|| Bool
llvm
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left (Width -> CallishMachOp
MO_U_Mul2     (DynFlags -> Width
wordWidth DynFlags
dflags))
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right [LocalReg] -> [CmmExpr] -> FCode ()
genericWordMul2Op
  PrimOp
FloatFabsOp -> \[CmmExpr]
_ -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if (Bool
ncg Bool -> Bool -> Bool
&& Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc) Bool -> Bool -> Bool
|| Bool
llvm
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left CallishMachOp
MO_F32_Fabs
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right (([LocalReg] -> [CmmExpr] -> FCode ())
 -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ()))
-> ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. (a -> b) -> a -> b
$ Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericFabsOp Width
W32
  PrimOp
DoubleFabsOp -> \[CmmExpr]
_ -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
OpDest_CallishHandledLater (Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
 -> OpDest)
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
-> OpDest
forall a b. (a -> b) -> a -> b
$
    if (Bool
ncg Bool -> Bool -> Bool
&& Bool
x86ish Bool -> Bool -> Bool
|| Bool
ppc) Bool -> Bool -> Bool
|| Bool
llvm
    then CallishMachOp
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. a -> Either a b
Left CallishMachOp
MO_F64_Fabs
    else ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. b -> Either a b
Right (([LocalReg] -> [CmmExpr] -> FCode ())
 -> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ()))
-> ([LocalReg] -> [CmmExpr] -> FCode ())
-> Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
forall a b. (a -> b) -> a -> b
$ Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericFabsOp Width
W64

  PrimOp
TagToEnumOp -> String -> [CmmExpr] -> OpDest
forall a. String -> a
panic String
"emitPrimOp: handled above in cgOpApp"

-- Out of line primops.
-- TODO compiler need not know about these

  PrimOp
UnsafeThawArrayOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CasArrayOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
UnsafeThawSmallArrayOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CasSmallArrayOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
NewPinnedByteArrayOp_Char -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
NewAlignedPinnedByteArrayOp_Char -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
MutableByteArrayIsPinnedOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
DoubleDecode_2IntOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
DoubleDecode_Int64Op -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
FloatDecode_IntOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ByteArrayIsPinnedOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ShrinkMutableByteArrayOp_Char -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ResizeMutableByteArrayOp_Char -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ShrinkSmallMutableArrayOp_Char -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
NewArrayArrayOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
NewMutVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
AtomicModifyMutVar2Op -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
AtomicModifyMutVar_Op -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CasMutVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CatchOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
RaiseOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
RaiseIOOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
MaskAsyncExceptionsOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
MaskUninterruptibleOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
UnmaskAsyncExceptionsOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
MaskStatus -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
AtomicallyOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
RetryOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CatchRetryOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CatchSTMOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
NewTVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ReadTVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ReadTVarIOOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
WriteTVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
NewMVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
TakeMVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
TryTakeMVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
PutMVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
TryPutMVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ReadMVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
TryReadMVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
IsEmptyMVarOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
DelayOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
WaitReadOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
WaitWriteOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ForkOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ForkOnOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
KillThreadOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
YieldOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
LabelThreadOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
IsCurrentThreadBoundOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
NoDuplicateOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ThreadStatusOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
MkWeakOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
MkWeakNoFinalizerOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
AddCFinalizerToWeakOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
DeRefWeakOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
FinalizeWeakOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
MakeStablePtrOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
DeRefStablePtrOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
MakeStableNameOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactNewOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactResizeOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactContainsOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactContainsAnyOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactGetFirstBlockOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactGetNextBlockOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactAllocateBlockOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactFixupPointersOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactAdd -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactAddWithSharing -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
CompactSize -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
SeqOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
GetSparkOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
NumSparks -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
DataToTagOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
MkApUpd0_Op -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
NewBCOOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
UnpackClosureOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ClosureSizeOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
GetApStackValOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
ClearCCSOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
TraceEventOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
TraceEventBinaryOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
TraceMarkerOp -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal
  PrimOp
SetThreadAllocationCounter -> [CmmExpr] -> OpDest
forall p. p -> OpDest
alwaysExternal

 where
  alwaysExternal :: p -> OpDest
alwaysExternal = \p
_ -> OpDest
OpDest_External
  -- Note [QuotRem optimization]
  -- ~~~~~~~~~~~~~~~~~~~~~~~~~~~
  --
  -- `quot` and `rem` with constant divisor can be implemented with fast bit-ops
  -- (shift, .&.).
  --
  -- Currently we only support optimization (performed in CmmOpt) when the
  -- constant is a power of 2. #9041 tracks the implementation of the general
  -- optimization.
  --
  -- `quotRem` can be optimized in the same way. However as it returns two values,
  -- it is implemented as a "callish" primop which is harder to match and
  -- to transform later on. For simplicity, the current implementation detects cases
  -- that can be optimized (see `quotRemCanBeOptimized`) and converts STG quotRem
  -- primop into two CMM quot and rem primops.
  quotRemCanBeOptimized :: [CmmExpr] -> Bool
quotRemCanBeOptimized = \case
    [CmmExpr
_, CmmLit (CmmInt Integer
n Width
_) ] -> Maybe Integer -> Bool
forall a. Maybe a -> Bool
isJust (Integer -> Maybe Integer
exactLog2 Integer
n)
    [CmmExpr]
_                         -> Bool
False

  ncg :: Bool
ncg = case DynFlags -> HscTarget
hscTarget DynFlags
dflags of
           HscTarget
HscAsm -> Bool
True
           HscTarget
_      -> Bool
False
  llvm :: Bool
llvm = case DynFlags -> HscTarget
hscTarget DynFlags
dflags of
           HscTarget
HscLlvm -> Bool
True
           HscTarget
_       -> Bool
False
  x86ish :: Bool
x86ish = case Platform -> Arch
platformArch (DynFlags -> Platform
targetPlatform DynFlags
dflags) of
             Arch
ArchX86    -> Bool
True
             Arch
ArchX86_64 -> Bool
True
             Arch
_          -> Bool
False
  ppc :: Bool
ppc = case Platform -> Arch
platformArch (DynFlags -> Platform
targetPlatform DynFlags
dflags) of
          Arch
ArchPPC      -> Bool
True
          ArchPPC_64 PPC_64ABI
_ -> Bool
True
          Arch
_            -> Bool
False

-- | Helper datatype used to ensure completion while keeping code smaller. Could
-- be totally eliminated in optimized builds.
data OpDest
  = OpDest_Nop
  | OpDest_Narrow !(Width -> Width -> MachOp, Width)
  -- | These primops are implemented by CallishMachOps, because they sometimes
  -- turn into foreign calls depending on the backend.
  | OpDest_Callish !CallishMachOp
  | OpDest_Translate !MachOp
  | OpDest_CallishHandledLater (Either CallishMachOp GenericOp)
  | OpDest_External
  -- | Basically a "manual" case, rather than one of the common repetitive forms
  -- above. The results are a parameter to the returned function so we know the
  -- choice of variant never depends on them.
  | OpDest_AllDone ([LocalReg] -- where to put the results
                    -> FCode ())

-- | Wrapper around '@dispatchPrimop@' which implements the cases represented
-- with '@OpDest@'.
--
-- Returns 'Nothing' if this primop should use its out-of-line implementation
-- (defined elsewhere) and 'Just' together with a code generating function that
-- takes the output regs as arguments otherwise.
emitPrimOp :: DynFlags
           -> PrimOp            -- the op
           -> [CmmExpr]         -- arguments
           -> Maybe ([LocalReg]        -- where to put the results
                      -> FCode ())

-- The rest just translate straightforwardly
emitPrimOp :: DynFlags -> PrimOp -> [CmmExpr] -> Maybe ([LocalReg] -> FCode ())
emitPrimOp DynFlags
dflags PrimOp
op [CmmExpr]
args = case DynFlags -> PrimOp -> [CmmExpr] -> OpDest
dispatchPrimop DynFlags
dflags PrimOp
op [CmmExpr]
args of
  OpDest
OpDest_Nop -> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a. a -> Maybe a
Just (([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ()))
-> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) CmmExpr
arg
    where [CmmExpr
arg] = [CmmExpr]
args

  OpDest_Narrow (Width -> Width -> MachOp
mop, Width
rep) -> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a. a -> Maybe a
Just (([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ()))
-> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (CmmExpr -> FCode ()) -> CmmExpr -> FCode ()
forall a b. (a -> b) -> a -> b
$
    MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> Width -> MachOp
mop Width
rep (DynFlags -> Width
wordWidth DynFlags
dflags)) [MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> Width -> MachOp
mop (DynFlags -> Width
wordWidth DynFlags
dflags) Width
rep) [CmmExpr
arg]]
    where [CmmExpr
arg] = [CmmExpr]
args

  OpDest_Callish CallishMachOp
prim -> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a. a -> Maybe a
Just (([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ()))
-> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall [LocalReg
res] CallishMachOp
prim [CmmExpr]
args

  OpDest_Translate MachOp
mop -> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a. a -> Maybe a
Just (([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ()))
-> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a b. (a -> b) -> a -> b
$ \[LocalReg
res] -> do
    let stmt :: CmmAGraph
stmt = CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp MachOp
mop [CmmExpr]
args)
    CmmAGraph -> FCode ()
emit CmmAGraph
stmt

  OpDest_CallishHandledLater Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
callOrNot -> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a. a -> Maybe a
Just (([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ()))
-> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a b. (a -> b) -> a -> b
$ \[LocalReg]
res0 -> case Either CallishMachOp ([LocalReg] -> [CmmExpr] -> FCode ())
callOrNot of
          Left CallishMachOp
op   -> CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ ForeignTarget -> [LocalReg] -> [CmmExpr] -> CmmAGraph
mkUnsafeCall (CallishMachOp -> ForeignTarget
PrimTarget CallishMachOp
op) [LocalReg]
res0 [CmmExpr]
args
          Right [LocalReg] -> [CmmExpr] -> FCode ()
gen -> [LocalReg] -> [CmmExpr] -> FCode ()
gen [LocalReg]
res0 [CmmExpr]
args

  OpDest_AllDone [LocalReg] -> FCode ()
f -> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a. a -> Maybe a
Just (([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ()))
-> ([LocalReg] -> FCode ()) -> Maybe ([LocalReg] -> FCode ())
forall a b. (a -> b) -> a -> b
$ [LocalReg] -> FCode ()
f

  OpDest
OpDest_External -> Maybe ([LocalReg] -> FCode ())
forall a. Maybe a
Nothing

type GenericOp = [CmmFormal] -> [CmmActual] -> FCode ()

genericIntQuotRemOp :: Width -> GenericOp
genericIntQuotRemOp :: Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericIntQuotRemOp Width
width [LocalReg
res_q, LocalReg
res_r] [CmmExpr
arg_x, CmmExpr
arg_y]
   = CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_q)
              (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_S_Quot Width
width) [CmmExpr
arg_x, CmmExpr
arg_y]) CmmAGraph -> CmmAGraph -> CmmAGraph
<*>
            CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r)
              (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_S_Rem  Width
width) [CmmExpr
arg_x, CmmExpr
arg_y])
genericIntQuotRemOp Width
_ [LocalReg]
_ [CmmExpr]
_ = String -> FCode ()
forall a. String -> a
panic String
"genericIntQuotRemOp"

genericWordQuotRemOp :: Width -> GenericOp
genericWordQuotRemOp :: Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericWordQuotRemOp Width
width [LocalReg
res_q, LocalReg
res_r] [CmmExpr
arg_x, CmmExpr
arg_y]
    = CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_q)
               (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_U_Quot Width
width) [CmmExpr
arg_x, CmmExpr
arg_y]) CmmAGraph -> CmmAGraph -> CmmAGraph
<*>
             CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r)
               (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_U_Rem  Width
width) [CmmExpr
arg_x, CmmExpr
arg_y])
genericWordQuotRemOp Width
_ [LocalReg]
_ [CmmExpr]
_ = String -> FCode ()
forall a. String -> a
panic String
"genericWordQuotRemOp"

genericWordQuotRem2Op :: DynFlags -> GenericOp
genericWordQuotRem2Op :: DynFlags -> [LocalReg] -> [CmmExpr] -> FCode ()
genericWordQuotRem2Op DynFlags
dflags [LocalReg
res_q, LocalReg
res_r] [CmmExpr
arg_x_high, CmmExpr
arg_x_low, CmmExpr
arg_y]
    = CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> FCode CmmAGraph -> FCode ()
forall (m :: * -> *) a b. Monad m => (a -> m b) -> m a -> m b
=<< Int -> CmmExpr -> CmmExpr -> CmmExpr -> FCode CmmAGraph
f (Width -> Int
widthInBits (DynFlags -> Width
wordWidth DynFlags
dflags)) CmmExpr
zero CmmExpr
arg_x_high CmmExpr
arg_x_low
    where    ty :: CmmType
ty = DynFlags -> CmmExpr -> CmmType
cmmExprType DynFlags
dflags CmmExpr
arg_x_high
             shl :: CmmExpr -> CmmExpr -> CmmExpr
shl   CmmExpr
x CmmExpr
i = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Shl   (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
i]
             shr :: CmmExpr -> CmmExpr -> CmmExpr
shr   CmmExpr
x CmmExpr
i = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_U_Shr (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
i]
             or :: CmmExpr -> CmmExpr -> CmmExpr
or    CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Or    (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
y]
             ge :: CmmExpr -> CmmExpr -> CmmExpr
ge    CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_U_Ge  (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
y]
             ne :: CmmExpr -> CmmExpr -> CmmExpr
ne    CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Ne    (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
y]
             minus :: CmmExpr -> CmmExpr -> CmmExpr
minus CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Sub   (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
y]
             times :: CmmExpr -> CmmExpr -> CmmExpr
times CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Mul   (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
y]
             zero :: CmmExpr
zero   = Integer -> CmmExpr
lit Integer
0
             one :: CmmExpr
one    = Integer -> CmmExpr
lit Integer
1
             negone :: CmmExpr
negone = Integer -> CmmExpr
lit (Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (Width -> Int
widthInBits (DynFlags -> Width
wordWidth DynFlags
dflags)) Integer -> Integer -> Integer
forall a. Num a => a -> a -> a
- Integer
1)
             lit :: Integer -> CmmExpr
lit Integer
i = CmmLit -> CmmExpr
CmmLit (Integer -> Width -> CmmLit
CmmInt Integer
i (DynFlags -> Width
wordWidth DynFlags
dflags))

             f :: Int -> CmmExpr -> CmmExpr -> CmmExpr -> FCode CmmAGraph
             f :: Int -> CmmExpr -> CmmExpr -> CmmExpr -> FCode CmmAGraph
f Int
0 CmmExpr
acc CmmExpr
high CmmExpr
_ = CmmAGraph -> FCode CmmAGraph
forall (m :: * -> *) a. Monad m => a -> m a
return (CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_q) CmmExpr
acc CmmAGraph -> CmmAGraph -> CmmAGraph
<*>
                                      CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) CmmExpr
high)
             f Int
i CmmExpr
acc CmmExpr
high CmmExpr
low =
                 do LocalReg
roverflowedBit <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
ty
                    LocalReg
rhigh'         <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
ty
                    LocalReg
rhigh''        <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
ty
                    LocalReg
rlow'          <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
ty
                    LocalReg
risge          <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
ty
                    LocalReg
racc'          <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
ty
                    let high' :: CmmExpr
high'         = CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
rhigh')
                        isge :: CmmExpr
isge          = CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
risge)
                        overflowedBit :: CmmExpr
overflowedBit = CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
roverflowedBit)
                    let this :: CmmAGraph
this = [CmmAGraph] -> CmmAGraph
catAGraphs
                               [CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
roverflowedBit)
                                          (CmmExpr -> CmmExpr -> CmmExpr
shr CmmExpr
high CmmExpr
negone),
                                CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
rhigh')
                                          (CmmExpr -> CmmExpr -> CmmExpr
or (CmmExpr -> CmmExpr -> CmmExpr
shl CmmExpr
high CmmExpr
one) (CmmExpr -> CmmExpr -> CmmExpr
shr CmmExpr
low CmmExpr
negone)),
                                CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
rlow')
                                          (CmmExpr -> CmmExpr -> CmmExpr
shl CmmExpr
low CmmExpr
one),
                                CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
risge)
                                          (CmmExpr -> CmmExpr -> CmmExpr
or (CmmExpr
overflowedBit CmmExpr -> CmmExpr -> CmmExpr
`ne` CmmExpr
zero)
                                              (CmmExpr
high' CmmExpr -> CmmExpr -> CmmExpr
`ge` CmmExpr
arg_y)),
                                CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
rhigh'')
                                          (CmmExpr
high' CmmExpr -> CmmExpr -> CmmExpr
`minus` (CmmExpr
arg_y CmmExpr -> CmmExpr -> CmmExpr
`times` CmmExpr
isge)),
                                CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
racc')
                                          (CmmExpr -> CmmExpr -> CmmExpr
or (CmmExpr -> CmmExpr -> CmmExpr
shl CmmExpr
acc CmmExpr
one) CmmExpr
isge)]
                    CmmAGraph
rest <- Int -> CmmExpr -> CmmExpr -> CmmExpr -> FCode CmmAGraph
f (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1) (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
racc'))
                                      (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
rhigh''))
                                      (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
rlow'))
                    CmmAGraph -> FCode CmmAGraph
forall (m :: * -> *) a. Monad m => a -> m a
return (CmmAGraph
this CmmAGraph -> CmmAGraph -> CmmAGraph
<*> CmmAGraph
rest)
genericWordQuotRem2Op DynFlags
_ [LocalReg]
_ [CmmExpr]
_ = String -> FCode ()
forall a. String -> a
panic String
"genericWordQuotRem2Op"

genericWordAdd2Op :: GenericOp
genericWordAdd2Op :: [LocalReg] -> [CmmExpr] -> FCode ()
genericWordAdd2Op [LocalReg
res_h, LocalReg
res_l] [CmmExpr
arg_x, CmmExpr
arg_y]
  = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
       LocalReg
r1 <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp (DynFlags -> CmmExpr -> CmmType
cmmExprType DynFlags
dflags CmmExpr
arg_x)
       LocalReg
r2 <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp (DynFlags -> CmmExpr -> CmmType
cmmExprType DynFlags
dflags CmmExpr
arg_x)
       let topHalf :: CmmExpr -> CmmExpr
topHalf CmmExpr
x = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_U_Shr (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
hww]
           toTopHalf :: CmmExpr -> CmmExpr
toTopHalf CmmExpr
x = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Shl (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
hww]
           bottomHalf :: CmmExpr -> CmmExpr
bottomHalf CmmExpr
x = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_And (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
hwm]
           add :: CmmExpr -> CmmExpr -> CmmExpr
add CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Add (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
y]
           or :: CmmExpr -> CmmExpr -> CmmExpr
or CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Or (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
y]
           hww :: CmmExpr
hww = CmmLit -> CmmExpr
CmmLit (Integer -> Width -> CmmLit
CmmInt (Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (Width -> Int
widthInBits (DynFlags -> Width
halfWordWidth DynFlags
dflags)))
                                (DynFlags -> Width
wordWidth DynFlags
dflags))
           hwm :: CmmExpr
hwm = CmmLit -> CmmExpr
CmmLit (Integer -> Width -> CmmLit
CmmInt (DynFlags -> Integer
halfWordMask DynFlags
dflags) (DynFlags -> Width
wordWidth DynFlags
dflags))
       CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ [CmmAGraph] -> CmmAGraph
catAGraphs
          [CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
r1)
               (CmmExpr -> CmmExpr -> CmmExpr
add (CmmExpr -> CmmExpr
bottomHalf CmmExpr
arg_x) (CmmExpr -> CmmExpr
bottomHalf CmmExpr
arg_y)),
           CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
r2)
               (CmmExpr -> CmmExpr -> CmmExpr
add (CmmExpr -> CmmExpr
topHalf (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
r1)))
                    (CmmExpr -> CmmExpr -> CmmExpr
add (CmmExpr -> CmmExpr
topHalf CmmExpr
arg_x) (CmmExpr -> CmmExpr
topHalf CmmExpr
arg_y))),
           CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_h)
               (CmmExpr -> CmmExpr
topHalf (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
r2))),
           CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_l)
               (CmmExpr -> CmmExpr -> CmmExpr
or (CmmExpr -> CmmExpr
toTopHalf (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
r2)))
                   (CmmExpr -> CmmExpr
bottomHalf (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
r1))))]
genericWordAdd2Op [LocalReg]
_ [CmmExpr]
_ = String -> FCode ()
forall a. String -> a
panic String
"genericWordAdd2Op"

-- | Implements branchless recovery of the carry flag @c@ by checking the
-- leftmost bits of both inputs @a@ and @b@ and result @r = a + b@:
--
-- @
--    c = a&b | (a|b)&~r
-- @
--
-- https://brodowsky.it-sky.net/2015/04/02/how-to-recover-the-carry-bit/
genericWordAddCOp :: GenericOp
genericWordAddCOp :: [LocalReg] -> [CmmExpr] -> FCode ()
genericWordAddCOp [LocalReg
res_r, LocalReg
res_c] [CmmExpr
aa, CmmExpr
bb]
 = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
      CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ [CmmAGraph] -> CmmAGraph
catAGraphs [
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordAdd DynFlags
dflags) [CmmExpr
aa,CmmExpr
bb]),
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_c) (CmmExpr -> CmmAGraph) -> CmmExpr -> CmmAGraph
forall a b. (a -> b) -> a -> b
$
          MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordUShr DynFlags
dflags) [
            MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordOr DynFlags
dflags) [
              MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordAnd DynFlags
dflags) [CmmExpr
aa,CmmExpr
bb],
              MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordAnd DynFlags
dflags) [
                MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordOr DynFlags
dflags) [CmmExpr
aa,CmmExpr
bb],
                MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordNot DynFlags
dflags) [CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
res_r)]
              ]
            ],
            DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
wORD_SIZE_IN_BITS DynFlags
dflags Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1)
          ]
        ]
genericWordAddCOp [LocalReg]
_ [CmmExpr]
_ = String -> FCode ()
forall a. String -> a
panic String
"genericWordAddCOp"

-- | Implements branchless recovery of the carry flag @c@ by checking the
-- leftmost bits of both inputs @a@ and @b@ and result @r = a - b@:
--
-- @
--    c = ~a&b | (~a|b)&r
-- @
--
-- https://brodowsky.it-sky.net/2015/04/02/how-to-recover-the-carry-bit/
genericWordSubCOp :: GenericOp
genericWordSubCOp :: [LocalReg] -> [CmmExpr] -> FCode ()
genericWordSubCOp [LocalReg
res_r, LocalReg
res_c] [CmmExpr
aa, CmmExpr
bb]
 = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
      CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ [CmmAGraph] -> CmmAGraph
catAGraphs [
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordSub DynFlags
dflags) [CmmExpr
aa,CmmExpr
bb]),
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_c) (CmmExpr -> CmmAGraph) -> CmmExpr -> CmmAGraph
forall a b. (a -> b) -> a -> b
$
          MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordUShr DynFlags
dflags) [
            MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordOr DynFlags
dflags) [
              MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordAnd DynFlags
dflags) [
                MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordNot DynFlags
dflags) [CmmExpr
aa],
                CmmExpr
bb
              ],
              MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordAnd DynFlags
dflags) [
                MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordOr DynFlags
dflags) [
                  MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordNot DynFlags
dflags) [CmmExpr
aa],
                  CmmExpr
bb
                ],
                CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
res_r)
              ]
            ],
            DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
wORD_SIZE_IN_BITS DynFlags
dflags Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1)
          ]
        ]
genericWordSubCOp [LocalReg]
_ [CmmExpr]
_ = String -> FCode ()
forall a. String -> a
panic String
"genericWordSubCOp"

genericIntAddCOp :: GenericOp
genericIntAddCOp :: [LocalReg] -> [CmmExpr] -> FCode ()
genericIntAddCOp [LocalReg
res_r, LocalReg
res_c] [CmmExpr
aa, CmmExpr
bb]
{-
   With some bit-twiddling, we can define int{Add,Sub}Czh portably in
   C, and without needing any comparisons.  This may not be the
   fastest way to do it - if you have better code, please send it! --SDM

   Return : r = a + b,  c = 0 if no overflow, 1 on overflow.

   We currently don't make use of the r value if c is != 0 (i.e.
   overflow), we just convert to big integers and try again.  This
   could be improved by making r and c the correct values for
   plugging into a new J#.

   { r = ((I_)(a)) + ((I_)(b));                                 \
     c = ((StgWord)(~(((I_)(a))^((I_)(b))) & (((I_)(a))^r)))    \
         >> (BITS_IN (I_) - 1);                                 \
   }
   Wading through the mass of bracketry, it seems to reduce to:
   c = ( (~(a^b)) & (a^r) ) >>unsigned (BITS_IN(I_)-1)

-}
 = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
      CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ [CmmAGraph] -> CmmAGraph
catAGraphs [
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordAdd DynFlags
dflags) [CmmExpr
aa,CmmExpr
bb]),
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_c) (CmmExpr -> CmmAGraph) -> CmmExpr -> CmmAGraph
forall a b. (a -> b) -> a -> b
$
          MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordUShr DynFlags
dflags) [
                MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordAnd DynFlags
dflags) [
                    MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordNot DynFlags
dflags) [MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordXor DynFlags
dflags) [CmmExpr
aa,CmmExpr
bb]],
                    MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordXor DynFlags
dflags) [CmmExpr
aa, CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
res_r)]
                ],
                DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
wORD_SIZE_IN_BITS DynFlags
dflags Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1)
          ]
        ]
genericIntAddCOp [LocalReg]
_ [CmmExpr]
_ = String -> FCode ()
forall a. String -> a
panic String
"genericIntAddCOp"

genericIntSubCOp :: GenericOp
genericIntSubCOp :: [LocalReg] -> [CmmExpr] -> FCode ()
genericIntSubCOp [LocalReg
res_r, LocalReg
res_c] [CmmExpr
aa, CmmExpr
bb]
{- Similarly:
   #define subIntCzh(r,c,a,b)                                   \
   { r = ((I_)(a)) - ((I_)(b));                                 \
     c = ((StgWord)((((I_)(a))^((I_)(b))) & (((I_)(a))^r)))     \
         >> (BITS_IN (I_) - 1);                                 \
   }

   c =  ((a^b) & (a^r)) >>unsigned (BITS_IN(I_)-1)
-}
 = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
      CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ [CmmAGraph] -> CmmAGraph
catAGraphs [
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordSub DynFlags
dflags) [CmmExpr
aa,CmmExpr
bb]),
        CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_c) (CmmExpr -> CmmAGraph) -> CmmExpr -> CmmAGraph
forall a b. (a -> b) -> a -> b
$
          MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordUShr DynFlags
dflags) [
                MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordAnd DynFlags
dflags) [
                    MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordXor DynFlags
dflags) [CmmExpr
aa,CmmExpr
bb],
                    MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordXor DynFlags
dflags) [CmmExpr
aa, CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
res_r)]
                ],
                DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
wORD_SIZE_IN_BITS DynFlags
dflags Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1)
          ]
        ]
genericIntSubCOp [LocalReg]
_ [CmmExpr]
_ = String -> FCode ()
forall a. String -> a
panic String
"genericIntSubCOp"

genericWordMul2Op :: GenericOp
genericWordMul2Op :: [LocalReg] -> [CmmExpr] -> FCode ()
genericWordMul2Op [LocalReg
res_h, LocalReg
res_l] [CmmExpr
arg_x, CmmExpr
arg_y]
 = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
      let t :: CmmType
t = DynFlags -> CmmExpr -> CmmType
cmmExprType DynFlags
dflags CmmExpr
arg_x
      CmmReg
xlyl <- (LocalReg -> CmmReg) -> FCode LocalReg -> FCode CmmReg
forall (m :: * -> *) a1 r. Monad m => (a1 -> r) -> m a1 -> m r
liftM LocalReg -> CmmReg
CmmLocal (FCode LocalReg -> FCode CmmReg) -> FCode LocalReg -> FCode CmmReg
forall a b. (a -> b) -> a -> b
$ CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
t
      CmmReg
xlyh <- (LocalReg -> CmmReg) -> FCode LocalReg -> FCode CmmReg
forall (m :: * -> *) a1 r. Monad m => (a1 -> r) -> m a1 -> m r
liftM LocalReg -> CmmReg
CmmLocal (FCode LocalReg -> FCode CmmReg) -> FCode LocalReg -> FCode CmmReg
forall a b. (a -> b) -> a -> b
$ CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
t
      CmmReg
xhyl <- (LocalReg -> CmmReg) -> FCode LocalReg -> FCode CmmReg
forall (m :: * -> *) a1 r. Monad m => (a1 -> r) -> m a1 -> m r
liftM LocalReg -> CmmReg
CmmLocal (FCode LocalReg -> FCode CmmReg) -> FCode LocalReg -> FCode CmmReg
forall a b. (a -> b) -> a -> b
$ CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
t
      CmmReg
r    <- (LocalReg -> CmmReg) -> FCode LocalReg -> FCode CmmReg
forall (m :: * -> *) a1 r. Monad m => (a1 -> r) -> m a1 -> m r
liftM LocalReg -> CmmReg
CmmLocal (FCode LocalReg -> FCode CmmReg) -> FCode LocalReg -> FCode CmmReg
forall a b. (a -> b) -> a -> b
$ CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
t
      -- This generic implementation is very simple and slow. We might
      -- well be able to do better, but for now this at least works.
      let topHalf :: CmmExpr -> CmmExpr
topHalf CmmExpr
x = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_U_Shr (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
hww]
          toTopHalf :: CmmExpr -> CmmExpr
toTopHalf CmmExpr
x = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Shl (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
hww]
          bottomHalf :: CmmExpr -> CmmExpr
bottomHalf CmmExpr
x = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_And (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
hwm]
          add :: CmmExpr -> CmmExpr -> CmmExpr
add CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Add (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
y]
          sum :: [CmmExpr] -> CmmExpr
sum = (CmmExpr -> CmmExpr -> CmmExpr) -> [CmmExpr] -> CmmExpr
forall (t :: * -> *) a. Foldable t => (a -> a -> a) -> t a -> a
foldl1 CmmExpr -> CmmExpr -> CmmExpr
add
          mul :: CmmExpr -> CmmExpr -> CmmExpr
mul CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Mul (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
y]
          or :: CmmExpr -> CmmExpr -> CmmExpr
or CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_Or (DynFlags -> Width
wordWidth DynFlags
dflags)) [CmmExpr
x, CmmExpr
y]
          hww :: CmmExpr
hww = CmmLit -> CmmExpr
CmmLit (Integer -> Width -> CmmLit
CmmInt (Int -> Integer
forall a b. (Integral a, Num b) => a -> b
fromIntegral (Width -> Int
widthInBits (DynFlags -> Width
halfWordWidth DynFlags
dflags)))
                               (DynFlags -> Width
wordWidth DynFlags
dflags))
          hwm :: CmmExpr
hwm = CmmLit -> CmmExpr
CmmLit (Integer -> Width -> CmmLit
CmmInt (DynFlags -> Integer
halfWordMask DynFlags
dflags) (DynFlags -> Width
wordWidth DynFlags
dflags))
      CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ [CmmAGraph] -> CmmAGraph
catAGraphs
             [CmmReg -> CmmExpr -> CmmAGraph
mkAssign CmmReg
xlyl
                  (CmmExpr -> CmmExpr -> CmmExpr
mul (CmmExpr -> CmmExpr
bottomHalf CmmExpr
arg_x) (CmmExpr -> CmmExpr
bottomHalf CmmExpr
arg_y)),
              CmmReg -> CmmExpr -> CmmAGraph
mkAssign CmmReg
xlyh
                  (CmmExpr -> CmmExpr -> CmmExpr
mul (CmmExpr -> CmmExpr
bottomHalf CmmExpr
arg_x) (CmmExpr -> CmmExpr
topHalf CmmExpr
arg_y)),
              CmmReg -> CmmExpr -> CmmAGraph
mkAssign CmmReg
xhyl
                  (CmmExpr -> CmmExpr -> CmmExpr
mul (CmmExpr -> CmmExpr
topHalf CmmExpr
arg_x) (CmmExpr -> CmmExpr
bottomHalf CmmExpr
arg_y)),
              CmmReg -> CmmExpr -> CmmAGraph
mkAssign CmmReg
r
                  ([CmmExpr] -> CmmExpr
sum [CmmExpr -> CmmExpr
topHalf    (CmmReg -> CmmExpr
CmmReg CmmReg
xlyl),
                        CmmExpr -> CmmExpr
bottomHalf (CmmReg -> CmmExpr
CmmReg CmmReg
xhyl),
                        CmmExpr -> CmmExpr
bottomHalf (CmmReg -> CmmExpr
CmmReg CmmReg
xlyh)]),
              CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_l)
                  (CmmExpr -> CmmExpr -> CmmExpr
or (CmmExpr -> CmmExpr
bottomHalf (CmmReg -> CmmExpr
CmmReg CmmReg
xlyl))
                      (CmmExpr -> CmmExpr
toTopHalf (CmmReg -> CmmExpr
CmmReg CmmReg
r))),
              CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_h)
                  ([CmmExpr] -> CmmExpr
sum [CmmExpr -> CmmExpr -> CmmExpr
mul (CmmExpr -> CmmExpr
topHalf CmmExpr
arg_x) (CmmExpr -> CmmExpr
topHalf CmmExpr
arg_y),
                        CmmExpr -> CmmExpr
topHalf (CmmReg -> CmmExpr
CmmReg CmmReg
xhyl),
                        CmmExpr -> CmmExpr
topHalf (CmmReg -> CmmExpr
CmmReg CmmReg
xlyh),
                        CmmExpr -> CmmExpr
topHalf (CmmReg -> CmmExpr
CmmReg CmmReg
r)])]
genericWordMul2Op [LocalReg]
_ [CmmExpr]
_ = String -> FCode ()
forall a. String -> a
panic String
"genericWordMul2Op"

-- This replicates what we had in libraries/base/GHC/Float.hs:
--
--    abs x    | x == 0    = 0 -- handles (-0.0)
--             | x >  0    = x
--             | otherwise = negateFloat x
genericFabsOp :: Width -> GenericOp
genericFabsOp :: Width -> [LocalReg] -> [CmmExpr] -> FCode ()
genericFabsOp Width
w [LocalReg
res_r] [CmmExpr
aa]
 = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
      let zero :: CmmExpr
zero   = CmmLit -> CmmExpr
CmmLit (Rational -> Width -> CmmLit
CmmFloat Rational
0 Width
w)

          eq :: CmmExpr -> CmmExpr -> CmmExpr
eq CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_F_Eq Width
w) [CmmExpr
x, CmmExpr
y]
          gt :: CmmExpr -> CmmExpr -> CmmExpr
gt CmmExpr
x CmmExpr
y = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_F_Gt Width
w) [CmmExpr
x, CmmExpr
y]

          neg :: CmmExpr -> CmmExpr
neg CmmExpr
x  = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> MachOp
MO_F_Neg Width
w) [CmmExpr
x]

          g1 :: CmmAGraph
g1 = [CmmAGraph] -> CmmAGraph
catAGraphs [CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) CmmExpr
zero]
          g2 :: CmmAGraph
g2 = [CmmAGraph] -> CmmAGraph
catAGraphs [CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) CmmExpr
aa]

      CmmReg
res_t <- LocalReg -> CmmReg
CmmLocal (LocalReg -> CmmReg) -> FCode LocalReg -> FCode CmmReg
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp (DynFlags -> CmmExpr -> CmmType
cmmExprType DynFlags
dflags CmmExpr
aa)
      let g3 :: CmmAGraph
g3 = [CmmAGraph] -> CmmAGraph
catAGraphs [CmmReg -> CmmExpr -> CmmAGraph
mkAssign CmmReg
res_t CmmExpr
aa,
                           CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) (CmmExpr -> CmmExpr
neg (CmmReg -> CmmExpr
CmmReg CmmReg
res_t))]

      CmmAGraph
g4 <- CmmExpr -> CmmAGraph -> CmmAGraph -> FCode CmmAGraph
mkCmmIfThenElse (CmmExpr -> CmmExpr -> CmmExpr
gt CmmExpr
aa CmmExpr
zero) CmmAGraph
g2 CmmAGraph
g3

      CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> FCode CmmAGraph -> FCode ()
forall (m :: * -> *) a b. Monad m => (a -> m b) -> m a -> m b
=<< CmmExpr -> CmmAGraph -> CmmAGraph -> FCode CmmAGraph
mkCmmIfThenElse (CmmExpr -> CmmExpr -> CmmExpr
eq CmmExpr
aa CmmExpr
zero) CmmAGraph
g1 CmmAGraph
g4

genericFabsOp Width
_ [LocalReg]
_ [CmmExpr]
_ = String -> FCode ()
forall a. String -> a
panic String
"genericFabsOp"

-- Note [Comparing stable names]
-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
--
-- A StableName# is actually a pointer to a stable name object (SNO)
-- containing an index into the stable name table (SNT). We
-- used to compare StableName#s by following the pointers to the
-- SNOs and checking whether they held the same SNT indices. However,
-- this is not necessary: there is a one-to-one correspondence
-- between SNOs and entries in the SNT, so simple pointer equality
-- does the trick.

------------------------------------------------------------------------------
-- Helpers for translating various minor variants of array indexing.

doIndexOffAddrOp :: Maybe MachOp
                 -> CmmType
                 -> [LocalReg]
                 -> [CmmExpr]
                 -> FCode ()
doIndexOffAddrOp :: Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOp Maybe MachOp
maybe_post_read_cast CmmType
rep [LocalReg
res] [CmmExpr
addr,CmmExpr
idx]
   = Int
-> Maybe MachOp
-> CmmType
-> LocalReg
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
mkBasicIndexedRead Int
0 Maybe MachOp
maybe_post_read_cast CmmType
rep LocalReg
res CmmExpr
addr CmmType
rep CmmExpr
idx
doIndexOffAddrOp Maybe MachOp
_ CmmType
_ [LocalReg]
_ [CmmExpr]
_
   = String -> FCode ()
forall a. String -> a
panic String
"GHC.StgToCmm.Prim: doIndexOffAddrOp"

doIndexOffAddrOpAs :: Maybe MachOp
                   -> CmmType
                   -> CmmType
                   -> [LocalReg]
                   -> [CmmExpr]
                   -> FCode ()
doIndexOffAddrOpAs :: Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexOffAddrOpAs Maybe MachOp
maybe_post_read_cast CmmType
rep CmmType
idx_rep [LocalReg
res] [CmmExpr
addr,CmmExpr
idx]
   = Int
-> Maybe MachOp
-> CmmType
-> LocalReg
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
mkBasicIndexedRead Int
0 Maybe MachOp
maybe_post_read_cast CmmType
rep LocalReg
res CmmExpr
addr CmmType
idx_rep CmmExpr
idx
doIndexOffAddrOpAs Maybe MachOp
_ CmmType
_ CmmType
_ [LocalReg]
_ [CmmExpr]
_
   = String -> FCode ()
forall a. String -> a
panic String
"GHC.StgToCmm.Prim: doIndexOffAddrOpAs"

doIndexByteArrayOp :: Maybe MachOp
                   -> CmmType
                   -> [LocalReg]
                   -> [CmmExpr]
                   -> FCode ()
doIndexByteArrayOp :: Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOp Maybe MachOp
maybe_post_read_cast CmmType
rep [LocalReg
res] [CmmExpr
addr,CmmExpr
idx]
   = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        Int
-> Maybe MachOp
-> CmmType
-> LocalReg
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
mkBasicIndexedRead (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags) Maybe MachOp
maybe_post_read_cast CmmType
rep LocalReg
res CmmExpr
addr CmmType
rep CmmExpr
idx
doIndexByteArrayOp Maybe MachOp
_ CmmType
_ [LocalReg]
_ [CmmExpr]
_
   = String -> FCode ()
forall a. String -> a
panic String
"GHC.StgToCmm.Prim: doIndexByteArrayOp"

doIndexByteArrayOpAs :: Maybe MachOp
                    -> CmmType
                    -> CmmType
                    -> [LocalReg]
                    -> [CmmExpr]
                    -> FCode ()
doIndexByteArrayOpAs :: Maybe MachOp
-> CmmType -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doIndexByteArrayOpAs Maybe MachOp
maybe_post_read_cast CmmType
rep CmmType
idx_rep [LocalReg
res] [CmmExpr
addr,CmmExpr
idx]
   = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        Int
-> Maybe MachOp
-> CmmType
-> LocalReg
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
mkBasicIndexedRead (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags) Maybe MachOp
maybe_post_read_cast CmmType
rep LocalReg
res CmmExpr
addr CmmType
idx_rep CmmExpr
idx
doIndexByteArrayOpAs Maybe MachOp
_ CmmType
_ CmmType
_ [LocalReg]
_ [CmmExpr]
_
   = String -> FCode ()
forall a. String -> a
panic String
"GHC.StgToCmm.Prim: doIndexByteArrayOpAs"

doReadPtrArrayOp :: LocalReg
                 -> CmmExpr
                 -> CmmExpr
                 -> FCode ()
doReadPtrArrayOp :: LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadPtrArrayOp LocalReg
res CmmExpr
addr CmmExpr
idx
   = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        Int
-> Maybe MachOp
-> CmmType
-> LocalReg
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
mkBasicIndexedRead (DynFlags -> Int
arrPtrsHdrSize DynFlags
dflags) Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
gcWord DynFlags
dflags) LocalReg
res CmmExpr
addr (DynFlags -> CmmType
gcWord DynFlags
dflags) CmmExpr
idx

doWriteOffAddrOp :: Maybe MachOp
                 -> CmmType
                 -> [LocalReg]
                 -> [CmmExpr]
                 -> FCode ()
doWriteOffAddrOp :: Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteOffAddrOp Maybe MachOp
maybe_pre_write_cast CmmType
idx_ty [] [CmmExpr
addr,CmmExpr
idx,CmmExpr
val]
   = Int
-> Maybe MachOp
-> CmmExpr
-> CmmType
-> CmmExpr
-> CmmExpr
-> FCode ()
mkBasicIndexedWrite Int
0 Maybe MachOp
maybe_pre_write_cast CmmExpr
addr CmmType
idx_ty CmmExpr
idx CmmExpr
val
doWriteOffAddrOp Maybe MachOp
_ CmmType
_ [LocalReg]
_ [CmmExpr]
_
   = String -> FCode ()
forall a. String -> a
panic String
"GHC.StgToCmm.Prim: doWriteOffAddrOp"

doWriteByteArrayOp :: Maybe MachOp
                   -> CmmType
                   -> [LocalReg]
                   -> [CmmExpr]
                   -> FCode ()
doWriteByteArrayOp :: Maybe MachOp -> CmmType -> [LocalReg] -> [CmmExpr] -> FCode ()
doWriteByteArrayOp Maybe MachOp
maybe_pre_write_cast CmmType
idx_ty [] [CmmExpr
addr,CmmExpr
idx,CmmExpr
val]
   = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        Int
-> Maybe MachOp
-> CmmExpr
-> CmmType
-> CmmExpr
-> CmmExpr
-> FCode ()
mkBasicIndexedWrite (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags) Maybe MachOp
maybe_pre_write_cast CmmExpr
addr CmmType
idx_ty CmmExpr
idx CmmExpr
val
doWriteByteArrayOp Maybe MachOp
_ CmmType
_ [LocalReg]
_ [CmmExpr]
_
   = String -> FCode ()
forall a. String -> a
panic String
"GHC.StgToCmm.Prim: doWriteByteArrayOp"

doWritePtrArrayOp :: CmmExpr
                  -> CmmExpr
                  -> CmmExpr
                  -> FCode ()
doWritePtrArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doWritePtrArrayOp CmmExpr
addr CmmExpr
idx CmmExpr
val
  = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
       let ty :: CmmType
ty = DynFlags -> CmmExpr -> CmmType
cmmExprType DynFlags
dflags CmmExpr
val
           hdr_size :: Int
hdr_size = DynFlags -> Int
arrPtrsHdrSize DynFlags
dflags
       -- Update remembered set for non-moving collector
       DynFlags -> FCode () -> FCode ()
forall a. DynFlags -> FCode a -> FCode ()
whenUpdRemSetEnabled DynFlags
dflags
           (FCode () -> FCode ()) -> FCode () -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmExpr -> FCode ()
emitUpdRemSetPush (DynFlags
-> Int -> CmmType -> CmmExpr -> CmmType -> CmmExpr -> CmmExpr
cmmLoadIndexOffExpr DynFlags
dflags Int
hdr_size CmmType
ty CmmExpr
addr CmmType
ty CmmExpr
idx)
       -- This write barrier is to ensure that the heap writes to the object
       -- referred to by val have happened before we write val into the array.
       -- See #12469 for details.
       [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall [] CallishMachOp
MO_WriteBarrier []
       Int
-> Maybe MachOp
-> CmmExpr
-> CmmType
-> CmmExpr
-> CmmExpr
-> FCode ()
mkBasicIndexedWrite Int
hdr_size Maybe MachOp
forall a. Maybe a
Nothing CmmExpr
addr CmmType
ty CmmExpr
idx CmmExpr
val
       CmmAGraph -> FCode ()
emit (CmmExpr -> CmmExpr -> CmmAGraph
setInfo CmmExpr
addr (CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel CLabel
mkMAP_DIRTY_infoLabel)))
       -- the write barrier.  We must write a byte into the mark table:
       -- bits8[a + header_size + StgMutArrPtrs_size(a) + x >> N]
       CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmExpr -> CmmExpr -> CmmAGraph
mkStore (
         DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExpr DynFlags
dflags
          (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExprW DynFlags
dflags (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
addr Int
hdr_size)
                         (DynFlags -> CmmExpr -> CmmExpr
loadArrPtrsSize DynFlags
dflags CmmExpr
addr))
          (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (DynFlags -> MachOp
mo_wordUShr DynFlags
dflags) [CmmExpr
idx,
                                           DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
mUT_ARR_PTRS_CARD_BITS DynFlags
dflags)])
         ) (CmmLit -> CmmExpr
CmmLit (Integer -> Width -> CmmLit
CmmInt Integer
1 Width
W8))

loadArrPtrsSize :: DynFlags -> CmmExpr -> CmmExpr
loadArrPtrsSize :: DynFlags -> CmmExpr -> CmmExpr
loadArrPtrsSize DynFlags
dflags CmmExpr
addr = CmmExpr -> CmmType -> CmmExpr
CmmLoad (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
addr Int
off) (DynFlags -> CmmType
bWord DynFlags
dflags)
 where off :: Int
off = DynFlags -> Int
fixedHdrSize DynFlags
dflags Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DynFlags -> Int
oFFSET_StgMutArrPtrs_ptrs DynFlags
dflags

mkBasicIndexedRead :: ByteOff      -- Initial offset in bytes
                   -> Maybe MachOp -- Optional result cast
                   -> CmmType      -- Type of element we are accessing
                   -> LocalReg     -- Destination
                   -> CmmExpr      -- Base address
                   -> CmmType      -- Type of element by which we are indexing
                   -> CmmExpr      -- Index
                   -> FCode ()
mkBasicIndexedRead :: Int
-> Maybe MachOp
-> CmmType
-> LocalReg
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
mkBasicIndexedRead Int
off Maybe MachOp
Nothing CmmType
ty LocalReg
res CmmExpr
base CmmType
idx_ty CmmExpr
idx
   = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (DynFlags
-> Int -> CmmType -> CmmExpr -> CmmType -> CmmExpr -> CmmExpr
cmmLoadIndexOffExpr DynFlags
dflags Int
off CmmType
ty CmmExpr
base CmmType
idx_ty CmmExpr
idx)
mkBasicIndexedRead Int
off (Just MachOp
cast) CmmType
ty LocalReg
res CmmExpr
base CmmType
idx_ty CmmExpr
idx
   = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp MachOp
cast [
                                   DynFlags
-> Int -> CmmType -> CmmExpr -> CmmType -> CmmExpr -> CmmExpr
cmmLoadIndexOffExpr DynFlags
dflags Int
off CmmType
ty CmmExpr
base CmmType
idx_ty CmmExpr
idx])

mkBasicIndexedWrite :: ByteOff      -- Initial offset in bytes
                    -> Maybe MachOp -- Optional value cast
                    -> CmmExpr      -- Base address
                    -> CmmType      -- Type of element by which we are indexing
                    -> CmmExpr      -- Index
                    -> CmmExpr      -- Value to write
                    -> FCode ()
mkBasicIndexedWrite :: Int
-> Maybe MachOp
-> CmmExpr
-> CmmType
-> CmmExpr
-> CmmExpr
-> FCode ()
mkBasicIndexedWrite Int
off Maybe MachOp
Nothing CmmExpr
base CmmType
idx_ty CmmExpr
idx CmmExpr
val
   = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        CmmExpr -> CmmExpr -> FCode ()
emitStore (DynFlags -> Int -> Width -> CmmExpr -> CmmExpr -> CmmExpr
cmmIndexOffExpr DynFlags
dflags Int
off (CmmType -> Width
typeWidth CmmType
idx_ty) CmmExpr
base CmmExpr
idx) CmmExpr
val
mkBasicIndexedWrite Int
off (Just MachOp
cast) CmmExpr
base CmmType
idx_ty CmmExpr
idx CmmExpr
val
   = Int
-> Maybe MachOp
-> CmmExpr
-> CmmType
-> CmmExpr
-> CmmExpr
-> FCode ()
mkBasicIndexedWrite Int
off Maybe MachOp
forall a. Maybe a
Nothing CmmExpr
base CmmType
idx_ty CmmExpr
idx (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp MachOp
cast [CmmExpr
val])

-- ----------------------------------------------------------------------------
-- Misc utils

cmmIndexOffExpr :: DynFlags
                -> ByteOff  -- Initial offset in bytes
                -> Width    -- Width of element by which we are indexing
                -> CmmExpr  -- Base address
                -> CmmExpr  -- Index
                -> CmmExpr
cmmIndexOffExpr :: DynFlags -> Int -> Width -> CmmExpr -> CmmExpr -> CmmExpr
cmmIndexOffExpr DynFlags
dflags Int
off Width
width CmmExpr
base CmmExpr
idx
   = DynFlags -> Width -> CmmExpr -> CmmExpr -> CmmExpr
cmmIndexExpr DynFlags
dflags Width
width (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
base Int
off) CmmExpr
idx

cmmLoadIndexOffExpr :: DynFlags
                    -> ByteOff  -- Initial offset in bytes
                    -> CmmType  -- Type of element we are accessing
                    -> CmmExpr  -- Base address
                    -> CmmType  -- Type of element by which we are indexing
                    -> CmmExpr  -- Index
                    -> CmmExpr
cmmLoadIndexOffExpr :: DynFlags
-> Int -> CmmType -> CmmExpr -> CmmType -> CmmExpr -> CmmExpr
cmmLoadIndexOffExpr DynFlags
dflags Int
off CmmType
ty CmmExpr
base CmmType
idx_ty CmmExpr
idx
   = CmmExpr -> CmmType -> CmmExpr
CmmLoad (DynFlags -> Int -> Width -> CmmExpr -> CmmExpr -> CmmExpr
cmmIndexOffExpr DynFlags
dflags Int
off (CmmType -> Width
typeWidth CmmType
idx_ty) CmmExpr
base CmmExpr
idx) CmmType
ty

setInfo :: CmmExpr -> CmmExpr -> CmmAGraph
setInfo :: CmmExpr -> CmmExpr -> CmmAGraph
setInfo CmmExpr
closure_ptr CmmExpr
info_ptr = CmmExpr -> CmmExpr -> CmmAGraph
mkStore CmmExpr
closure_ptr CmmExpr
info_ptr

------------------------------------------------------------------------------
-- Helpers for translating vector primops.

vecVmmType :: PrimOpVecCat -> Length -> Width -> CmmType
vecVmmType :: PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
pocat Int
n Width
w = Int -> CmmType -> CmmType
vec Int
n (PrimOpVecCat -> Width -> CmmType
vecCmmCat PrimOpVecCat
pocat Width
w)

vecCmmCat :: PrimOpVecCat -> Width -> CmmType
vecCmmCat :: PrimOpVecCat -> Width -> CmmType
vecCmmCat PrimOpVecCat
IntVec   = Width -> CmmType
cmmBits
vecCmmCat PrimOpVecCat
WordVec  = Width -> CmmType
cmmBits
vecCmmCat PrimOpVecCat
FloatVec = Width -> CmmType
cmmFloat

vecElemInjectCast :: DynFlags -> PrimOpVecCat -> Width -> Maybe MachOp
vecElemInjectCast :: DynFlags -> PrimOpVecCat -> Width -> Maybe MachOp
vecElemInjectCast DynFlags
_      PrimOpVecCat
FloatVec Width
_   =  Maybe MachOp
forall a. Maybe a
Nothing
vecElemInjectCast DynFlags
dflags PrimOpVecCat
IntVec   Width
W8  =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo8  DynFlags
dflags)
vecElemInjectCast DynFlags
dflags PrimOpVecCat
IntVec   Width
W16 =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo16 DynFlags
dflags)
vecElemInjectCast DynFlags
dflags PrimOpVecCat
IntVec   Width
W32 =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)
vecElemInjectCast DynFlags
_      PrimOpVecCat
IntVec   Width
W64 =  Maybe MachOp
forall a. Maybe a
Nothing
vecElemInjectCast DynFlags
dflags PrimOpVecCat
WordVec  Width
W8  =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo8  DynFlags
dflags)
vecElemInjectCast DynFlags
dflags PrimOpVecCat
WordVec  Width
W16 =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo16 DynFlags
dflags)
vecElemInjectCast DynFlags
dflags PrimOpVecCat
WordVec  Width
W32 =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_WordTo32 DynFlags
dflags)
vecElemInjectCast DynFlags
_      PrimOpVecCat
WordVec  Width
W64 =  Maybe MachOp
forall a. Maybe a
Nothing
vecElemInjectCast DynFlags
_      PrimOpVecCat
_        Width
_   =  Maybe MachOp
forall a. Maybe a
Nothing

vecElemProjectCast :: DynFlags -> PrimOpVecCat -> Width -> Maybe MachOp
vecElemProjectCast :: DynFlags -> PrimOpVecCat -> Width -> Maybe MachOp
vecElemProjectCast DynFlags
_      PrimOpVecCat
FloatVec Width
_   =  Maybe MachOp
forall a. Maybe a
Nothing
vecElemProjectCast DynFlags
dflags PrimOpVecCat
IntVec   Width
W8  =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_8ToWord  DynFlags
dflags)
vecElemProjectCast DynFlags
dflags PrimOpVecCat
IntVec   Width
W16 =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_16ToWord DynFlags
dflags)
vecElemProjectCast DynFlags
dflags PrimOpVecCat
IntVec   Width
W32 =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_s_32ToWord DynFlags
dflags)
vecElemProjectCast DynFlags
_      PrimOpVecCat
IntVec   Width
W64 =  Maybe MachOp
forall a. Maybe a
Nothing
vecElemProjectCast DynFlags
dflags PrimOpVecCat
WordVec  Width
W8  =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_8ToWord  DynFlags
dflags)
vecElemProjectCast DynFlags
dflags PrimOpVecCat
WordVec  Width
W16 =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_16ToWord DynFlags
dflags)
vecElemProjectCast DynFlags
dflags PrimOpVecCat
WordVec  Width
W32 =  MachOp -> Maybe MachOp
forall a. a -> Maybe a
Just (DynFlags -> MachOp
mo_u_32ToWord DynFlags
dflags)
vecElemProjectCast DynFlags
_      PrimOpVecCat
WordVec  Width
W64 =  Maybe MachOp
forall a. Maybe a
Nothing
vecElemProjectCast DynFlags
_      PrimOpVecCat
_        Width
_   =  Maybe MachOp
forall a. Maybe a
Nothing


-- NOTE [SIMD Design for the future]
-- Check to make sure that we can generate code for the specified vector type
-- given the current set of dynamic flags.
-- Currently these checks are specific to x86 and x86_64 architecture.
-- This should be fixed!
-- In particular,
-- 1) Add better support for other architectures! (this may require a redesign)
-- 2) Decouple design choices from LLVM's pseudo SIMD model!
--   The high level LLVM naive rep makes per CPU family SIMD generation is own
--   optimization problem, and hides important differences in eg ARM vs x86_64 simd
-- 3) Depending on the architecture, the SIMD registers may also support general
--    computations on Float/Double/Word/Int scalars, but currently on
--    for example x86_64, we always put Word/Int (or sized) in GPR
--    (general purpose) registers. Would relaxing that allow for
--    useful optimization opportunities?
--      Phrased differently, it is worth experimenting with supporting
--    different register mapping strategies than we currently have, especially if
--    someday we want SIMD to be a first class denizen in GHC along with scalar
--    values!
--      The current design with respect to register mapping of scalars could
--    very well be the best,but exploring the  design space and doing careful
--    measurments is the only only way to validate that.
--      In some next generation CPU ISAs, notably RISC V, the SIMD extension
--    includes  support for a sort of run time CPU dependent vectorization parameter,
--    where a loop may act upon a single scalar each iteration OR some 2,4,8 ...
--    element chunk! Time will tell if that direction sees wide adoption,
--    but it is from that context that unifying our handling of simd and scalars
--    may benefit. It is not likely to benefit current architectures, though
--    it may very well be a design perspective that helps guide improving the NCG.


checkVecCompatibility :: DynFlags -> PrimOpVecCat -> Length -> Width -> FCode ()
checkVecCompatibility :: DynFlags -> PrimOpVecCat -> Int -> Width -> FCode ()
checkVecCompatibility DynFlags
dflags PrimOpVecCat
vcat Int
l Width
w = do
    Bool -> FCode () -> FCode ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (DynFlags -> HscTarget
hscTarget DynFlags
dflags HscTarget -> HscTarget -> Bool
forall a. Eq a => a -> a -> Bool
/= HscTarget
HscLlvm) (FCode () -> FCode ()) -> FCode () -> FCode ()
forall a b. (a -> b) -> a -> b
$ do
        String -> FCode ()
forall a. String -> a
sorry (String -> FCode ()) -> String -> FCode ()
forall a b. (a -> b) -> a -> b
$ [String] -> String
unlines [String
"SIMD vector instructions require the LLVM back-end."
                         ,String
"Please use -fllvm."]
    Width -> PrimOpVecCat -> Int -> Width -> FCode ()
check Width
vecWidth PrimOpVecCat
vcat Int
l Width
w
  where
    check :: Width -> PrimOpVecCat -> Length -> Width -> FCode ()
    check :: Width -> PrimOpVecCat -> Int -> Width -> FCode ()
check Width
W128 PrimOpVecCat
FloatVec Int
4 Width
W32 | Bool -> Bool
not (DynFlags -> Bool
isSseEnabled DynFlags
dflags) =
        String -> FCode ()
forall a. String -> a
sorry (String -> FCode ()) -> String -> FCode ()
forall a b. (a -> b) -> a -> b
$ String
"128-bit wide single-precision floating point " String -> String -> String
forall a. [a] -> [a] -> [a]
++
                String
"SIMD vector instructions require at least -msse."
    check Width
W128 PrimOpVecCat
_ Int
_ Width
_ | Bool -> Bool
not (DynFlags -> Bool
isSse2Enabled DynFlags
dflags) =
        String -> FCode ()
forall a. String -> a
sorry (String -> FCode ()) -> String -> FCode ()
forall a b. (a -> b) -> a -> b
$ String
"128-bit wide integer and double precision " String -> String -> String
forall a. [a] -> [a] -> [a]
++
                String
"SIMD vector instructions require at least -msse2."
    check Width
W256 PrimOpVecCat
FloatVec Int
_ Width
_ | Bool -> Bool
not (DynFlags -> Bool
isAvxEnabled DynFlags
dflags) =
        String -> FCode ()
forall a. String -> a
sorry (String -> FCode ()) -> String -> FCode ()
forall a b. (a -> b) -> a -> b
$ String
"256-bit wide floating point " String -> String -> String
forall a. [a] -> [a] -> [a]
++
                String
"SIMD vector instructions require at least -mavx."
    check Width
W256 PrimOpVecCat
_ Int
_ Width
_ | Bool -> Bool
not (DynFlags -> Bool
isAvx2Enabled DynFlags
dflags) =
        String -> FCode ()
forall a. String -> a
sorry (String -> FCode ()) -> String -> FCode ()
forall a b. (a -> b) -> a -> b
$ String
"256-bit wide integer " String -> String -> String
forall a. [a] -> [a] -> [a]
++
                String
"SIMD vector instructions require at least -mavx2."
    check Width
W512 PrimOpVecCat
_ Int
_ Width
_ | Bool -> Bool
not (DynFlags -> Bool
isAvx512fEnabled DynFlags
dflags) =
        String -> FCode ()
forall a. String -> a
sorry (String -> FCode ()) -> String -> FCode ()
forall a b. (a -> b) -> a -> b
$ String
"512-bit wide " String -> String -> String
forall a. [a] -> [a] -> [a]
++
                String
"SIMD vector instructions require -mavx512f."
    check Width
_ PrimOpVecCat
_ Int
_ Width
_ = () -> FCode ()
forall (m :: * -> *) a. Monad m => a -> m a
return ()

    vecWidth :: Width
vecWidth = CmmType -> Width
typeWidth (PrimOpVecCat -> Int -> Width -> CmmType
vecVmmType PrimOpVecCat
vcat Int
l Width
w)

------------------------------------------------------------------------------
-- Helpers for translating vector packing and unpacking.

doVecPackOp :: Maybe MachOp  -- Cast from element to vector component
            -> CmmType       -- Type of vector
            -> CmmExpr       -- Initial vector
            -> [CmmExpr]     -- Elements
            -> CmmFormal     -- Destination for result
            -> FCode ()
doVecPackOp :: Maybe MachOp
-> CmmType -> CmmExpr -> [CmmExpr] -> LocalReg -> FCode ()
doVecPackOp Maybe MachOp
maybe_pre_write_cast CmmType
ty CmmExpr
z [CmmExpr]
es LocalReg
res = do
    LocalReg
dst <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
ty
    CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
dst) CmmExpr
z
    LocalReg -> [CmmExpr] -> Int -> FCode ()
vecPack LocalReg
dst [CmmExpr]
es Int
0
  where
    vecPack :: CmmFormal -> [CmmExpr] -> Int -> FCode ()
    vecPack :: LocalReg -> [CmmExpr] -> Int -> FCode ()
vecPack LocalReg
src [] Int
_ =
        CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
src))

    vecPack LocalReg
src (CmmExpr
e : [CmmExpr]
es) Int
i = do
        LocalReg
dst <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
ty
        if CmmType -> Bool
isFloatType (CmmType -> CmmType
vecElemType CmmType
ty)
          then CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
dst) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Int -> Width -> MachOp
MO_VF_Insert Int
len Width
wid)
                                                    [CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
src), CmmExpr -> CmmExpr
cast CmmExpr
e, CmmExpr
iLit])
          else CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
dst) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Int -> Width -> MachOp
MO_V_Insert Int
len Width
wid)
                                                    [CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
src), CmmExpr -> CmmExpr
cast CmmExpr
e, CmmExpr
iLit])
        LocalReg -> [CmmExpr] -> Int -> FCode ()
vecPack LocalReg
dst [CmmExpr]
es (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1)
      where
        -- vector indices are always 32-bits
        iLit :: CmmExpr
iLit = CmmLit -> CmmExpr
CmmLit (Integer -> Width -> CmmLit
CmmInt (Int -> Integer
forall a. Integral a => a -> Integer
toInteger Int
i) Width
W32)

    cast :: CmmExpr -> CmmExpr
    cast :: CmmExpr -> CmmExpr
cast CmmExpr
val = case Maybe MachOp
maybe_pre_write_cast of
                 Maybe MachOp
Nothing   -> CmmExpr
val
                 Just MachOp
cast -> MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp MachOp
cast [CmmExpr
val]

    len :: Length
    len :: Int
len = CmmType -> Int
vecLength CmmType
ty

    wid :: Width
    wid :: Width
wid = CmmType -> Width
typeWidth (CmmType -> CmmType
vecElemType CmmType
ty)

doVecUnpackOp :: Maybe MachOp  -- Cast from vector component to element result
              -> CmmType       -- Type of vector
              -> CmmExpr       -- Vector
              -> [CmmFormal]   -- Element results
              -> FCode ()
doVecUnpackOp :: Maybe MachOp -> CmmType -> CmmExpr -> [LocalReg] -> FCode ()
doVecUnpackOp Maybe MachOp
maybe_post_read_cast CmmType
ty CmmExpr
e [LocalReg]
res =
    [LocalReg] -> Int -> FCode ()
vecUnpack [LocalReg]
res Int
0
  where
    vecUnpack :: [CmmFormal] -> Int -> FCode ()
    vecUnpack :: [LocalReg] -> Int -> FCode ()
vecUnpack [] Int
_ =
        () -> FCode ()
forall (m :: * -> *) a. Monad m => a -> m a
return ()

    vecUnpack (LocalReg
r : [LocalReg]
rs) Int
i = do
        if CmmType -> Bool
isFloatType (CmmType -> CmmType
vecElemType CmmType
ty)
          then CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
r) (CmmExpr -> CmmExpr
cast (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Int -> Width -> MachOp
MO_VF_Extract Int
len Width
wid)
                                             [CmmExpr
e, CmmExpr
iLit]))
          else CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
r) (CmmExpr -> CmmExpr
cast (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Int -> Width -> MachOp
MO_V_Extract Int
len Width
wid)
                                             [CmmExpr
e, CmmExpr
iLit]))
        [LocalReg] -> Int -> FCode ()
vecUnpack [LocalReg]
rs (Int
i Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
1)
      where
        -- vector indices are always 32-bits
        iLit :: CmmExpr
iLit = CmmLit -> CmmExpr
CmmLit (Integer -> Width -> CmmLit
CmmInt (Int -> Integer
forall a. Integral a => a -> Integer
toInteger Int
i) Width
W32)

    cast :: CmmExpr -> CmmExpr
    cast :: CmmExpr -> CmmExpr
cast CmmExpr
val = case Maybe MachOp
maybe_post_read_cast of
                 Maybe MachOp
Nothing   -> CmmExpr
val
                 Just MachOp
cast -> MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp MachOp
cast [CmmExpr
val]

    len :: Length
    len :: Int
len = CmmType -> Int
vecLength CmmType
ty

    wid :: Width
    wid :: Width
wid = CmmType -> Width
typeWidth (CmmType -> CmmType
vecElemType CmmType
ty)

doVecInsertOp :: Maybe MachOp  -- Cast from element to vector component
              -> CmmType       -- Vector type
              -> CmmExpr       -- Source vector
              -> CmmExpr       -- Element
              -> CmmExpr       -- Index at which to insert element
              -> CmmFormal     -- Destination for result
              -> FCode ()
doVecInsertOp :: Maybe MachOp
-> CmmType -> CmmExpr -> CmmExpr -> CmmExpr -> LocalReg -> FCode ()
doVecInsertOp Maybe MachOp
maybe_pre_write_cast CmmType
ty CmmExpr
src CmmExpr
e CmmExpr
idx LocalReg
res = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    -- vector indices are always 32-bits
    let idx' :: CmmExpr
        idx' :: CmmExpr
idx' = MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Width -> Width -> MachOp
MO_SS_Conv (DynFlags -> Width
wordWidth DynFlags
dflags) Width
W32) [CmmExpr
idx]
    if CmmType -> Bool
isFloatType (CmmType -> CmmType
vecElemType CmmType
ty)
      then CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Int -> Width -> MachOp
MO_VF_Insert Int
len Width
wid) [CmmExpr
src, CmmExpr -> CmmExpr
cast CmmExpr
e, CmmExpr
idx'])
      else CmmReg -> CmmExpr -> FCode ()
emitAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp (Int -> Width -> MachOp
MO_V_Insert Int
len Width
wid) [CmmExpr
src, CmmExpr -> CmmExpr
cast CmmExpr
e, CmmExpr
idx'])
  where
    cast :: CmmExpr -> CmmExpr
    cast :: CmmExpr -> CmmExpr
cast CmmExpr
val = case Maybe MachOp
maybe_pre_write_cast of
                 Maybe MachOp
Nothing   -> CmmExpr
val
                 Just MachOp
cast -> MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp MachOp
cast [CmmExpr
val]

    len :: Length
    len :: Int
len = CmmType -> Int
vecLength CmmType
ty

    wid :: Width
    wid :: Width
wid = CmmType -> Width
typeWidth (CmmType -> CmmType
vecElemType CmmType
ty)

------------------------------------------------------------------------------
-- Helpers for translating prefetching.


-- | Translate byte array prefetch operations into proper primcalls.
doPrefetchByteArrayOp :: Int
                      -> [CmmExpr]
                      -> FCode ()
doPrefetchByteArrayOp :: Int -> [CmmExpr] -> FCode ()
doPrefetchByteArrayOp Int
locality  [CmmExpr
addr,CmmExpr
idx]
   = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        Int -> Int -> CmmExpr -> CmmExpr -> FCode ()
mkBasicPrefetch Int
locality (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)  CmmExpr
addr CmmExpr
idx
doPrefetchByteArrayOp Int
_ [CmmExpr]
_
   = String -> FCode ()
forall a. String -> a
panic String
"GHC.StgToCmm.Prim: doPrefetchByteArrayOp"

-- | Translate mutable byte array prefetch operations into proper primcalls.
doPrefetchMutableByteArrayOp :: Int
                      -> [CmmExpr]
                      -> FCode ()
doPrefetchMutableByteArrayOp :: Int -> [CmmExpr] -> FCode ()
doPrefetchMutableByteArrayOp Int
locality  [CmmExpr
addr,CmmExpr
idx]
   = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        Int -> Int -> CmmExpr -> CmmExpr -> FCode ()
mkBasicPrefetch Int
locality (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)  CmmExpr
addr CmmExpr
idx
doPrefetchMutableByteArrayOp Int
_ [CmmExpr]
_
   = String -> FCode ()
forall a. String -> a
panic String
"GHC.StgToCmm.Prim: doPrefetchByteArrayOp"

-- | Translate address prefetch operations into proper primcalls.
doPrefetchAddrOp ::Int
                 -> [CmmExpr]
                 -> FCode ()
doPrefetchAddrOp :: Int -> [CmmExpr] -> FCode ()
doPrefetchAddrOp Int
locality   [CmmExpr
addr,CmmExpr
idx]
   = Int -> Int -> CmmExpr -> CmmExpr -> FCode ()
mkBasicPrefetch Int
locality Int
0  CmmExpr
addr CmmExpr
idx
doPrefetchAddrOp Int
_ [CmmExpr]
_
   = String -> FCode ()
forall a. String -> a
panic String
"GHC.StgToCmm.Prim: doPrefetchAddrOp"

-- | Translate value prefetch operations into proper primcalls.
doPrefetchValueOp :: Int
                 -> [CmmExpr]
                 -> FCode ()
doPrefetchValueOp :: Int -> [CmmExpr] -> FCode ()
doPrefetchValueOp  Int
locality   [CmmExpr
addr]
  =  do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        Int -> Int -> CmmExpr -> CmmExpr -> FCode ()
mkBasicPrefetch Int
locality Int
0 CmmExpr
addr  (CmmLit -> CmmExpr
CmmLit (Integer -> Width -> CmmLit
CmmInt Integer
0 (DynFlags -> Width
wordWidth DynFlags
dflags)))
doPrefetchValueOp Int
_ [CmmExpr]
_
  = String -> FCode ()
forall a. String -> a
panic String
"GHC.StgToCmm.Prim: doPrefetchValueOp"

-- | helper to generate prefetch primcalls
mkBasicPrefetch :: Int          -- Locality level 0-3
                -> ByteOff      -- Initial offset in bytes
                -> CmmExpr      -- Base address
                -> CmmExpr      -- Index
                -> FCode ()
mkBasicPrefetch :: Int -> Int -> CmmExpr -> CmmExpr -> FCode ()
mkBasicPrefetch Int
locality Int
off CmmExpr
base CmmExpr
idx
   = do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall [] (Int -> CallishMachOp
MO_Prefetch_Data Int
locality) [DynFlags -> Width -> CmmExpr -> CmmExpr -> CmmExpr
cmmIndexExpr DynFlags
dflags Width
W8 (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
base Int
off) CmmExpr
idx]
        () -> FCode ()
forall (m :: * -> *) a. Monad m => a -> m a
return ()

-- ----------------------------------------------------------------------------
-- Allocating byte arrays

-- | Takes a register to return the newly allocated array in and the
-- size of the new array in bytes. Allocates a new
-- 'MutableByteArray#'.
doNewByteArrayOp :: CmmFormal -> ByteOff -> FCode ()
doNewByteArrayOp :: LocalReg -> Int -> FCode ()
doNewByteArrayOp LocalReg
res_r Int
n = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags

    let info_ptr :: CmmExpr
info_ptr = CLabel -> CmmExpr
mkLblExpr CLabel
mkArrWords_infoLabel
        rep :: SMRep
rep = DynFlags -> Int -> SMRep
arrWordsRep DynFlags
dflags Int
n

    CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
tickyAllocPrim (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags))
        (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> SMRep -> Int
nonHdrSize DynFlags
dflags SMRep
rep))
        (DynFlags -> CmmExpr
zeroExpr DynFlags
dflags)

    let hdr_size :: Int
hdr_size = DynFlags -> Int
fixedHdrSize DynFlags
dflags

    CmmExpr
base <- SMRep -> CmmExpr -> CmmExpr -> [(CmmExpr, Int)] -> FCode CmmExpr
allocHeapClosure SMRep
rep CmmExpr
info_ptr CmmExpr
cccsExpr
                     [ (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
n,
                        Int
hdr_size Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DynFlags -> Int
oFFSET_StgArrBytes_bytes DynFlags
dflags)
                     ]

    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) CmmExpr
base

-- ----------------------------------------------------------------------------
-- Comparing byte arrays

doCompareByteArraysOp :: LocalReg -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr
                     -> FCode ()
doCompareByteArraysOp :: LocalReg
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCompareByteArraysOp LocalReg
res CmmExpr
ba1 CmmExpr
ba1_off CmmExpr
ba2 CmmExpr
ba2_off CmmExpr
n = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    CmmExpr
ba1_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExpr DynFlags
dflags (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
ba1 (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)) CmmExpr
ba1_off
    CmmExpr
ba2_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExpr DynFlags
dflags (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
ba2 (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)) CmmExpr
ba2_off

    -- short-cut in case of equal pointers avoiding a costly
    -- subroutine call to the memcmp(3) routine; the Cmm logic below
    -- results in assembly code being generated for
    --
    --   cmpPrefix10 :: ByteArray# -> ByteArray# -> Int#
    --   cmpPrefix10 ba1 ba2 = compareByteArrays# ba1 0# ba2 0# 10#
    --
    -- that looks like
    --
    --          leaq 16(%r14),%rax
    --          leaq 16(%rsi),%rbx
    --          xorl %ecx,%ecx
    --          cmpq %rbx,%rax
    --          je l_ptr_eq
    --
    --          ; NB: the common case (unequal pointers) falls-through
    --          ; the conditional jump, and therefore matches the
    --          ; usual static branch prediction convention of modern cpus
    --
    --          subq $8,%rsp
    --          movq %rbx,%rsi
    --          movq %rax,%rdi
    --          movl $10,%edx
    --          xorl %eax,%eax
    --          call memcmp
    --          addq $8,%rsp
    --          movslq %eax,%rax
    --          movq %rax,%rcx
    --  l_ptr_eq:
    --          movq %rcx,%rbx
    --          jmp *(%rbp)

    BlockId
l_ptr_eq <- FCode BlockId
forall (m :: * -> *). MonadUnique m => m BlockId
newBlockId
    BlockId
l_ptr_ne <- FCode BlockId
forall (m :: * -> *). MonadUnique m => m BlockId
newBlockId

    CmmAGraph -> FCode ()
emit (CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res) (DynFlags -> CmmExpr
zeroExpr DynFlags
dflags))
    CmmAGraph -> FCode ()
emit (CmmExpr -> BlockId -> BlockId -> Maybe Bool -> CmmAGraph
mkCbranch (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmEqWord DynFlags
dflags CmmExpr
ba1_p CmmExpr
ba2_p)
                    BlockId
l_ptr_eq BlockId
l_ptr_ne (Bool -> Maybe Bool
forall a. a -> Maybe a
Just Bool
False))

    BlockId -> FCode ()
emitLabel BlockId
l_ptr_ne
    LocalReg -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitMemcmpCall LocalReg
res CmmExpr
ba1_p CmmExpr
ba2_p CmmExpr
n Int
1

    BlockId -> FCode ()
emitLabel BlockId
l_ptr_eq

-- ----------------------------------------------------------------------------
-- Copying byte arrays

-- | Takes a source 'ByteArray#', an offset in the source array, a
-- destination 'MutableByteArray#', an offset into the destination
-- array, and the number of bytes to copy.  Copies the given number of
-- bytes from the source array to the destination array.
doCopyByteArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr
                  -> FCode ()
doCopyByteArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyByteArrayOp = (CmmExpr
 -> CmmExpr
 -> CmmExpr
 -> CmmExpr
 -> CmmExpr
 -> Alignment
 -> FCode ())
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
emitCopyByteArray CmmExpr
-> CmmExpr
-> CmmExpr
-> CmmExpr
-> CmmExpr
-> Alignment
-> FCode ()
forall p p.
p -> p -> CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
copy
  where
    -- Copy data (we assume the arrays aren't overlapping since
    -- they're of different types)
    copy :: p -> p -> CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
copy p
_src p
_dst CmmExpr
dst_p CmmExpr
src_p CmmExpr
bytes Alignment
align =
        CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall CmmExpr
dst_p CmmExpr
src_p CmmExpr
bytes Alignment
align

-- | Takes a source 'MutableByteArray#', an offset in the source
-- array, a destination 'MutableByteArray#', an offset into the
-- destination array, and the number of bytes to copy.  Copies the
-- given number of bytes from the source array to the destination
-- array.
doCopyMutableByteArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr
                         -> FCode ()
doCopyMutableByteArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyMutableByteArrayOp = (CmmExpr
 -> CmmExpr
 -> CmmExpr
 -> CmmExpr
 -> CmmExpr
 -> Alignment
 -> FCode ())
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
emitCopyByteArray CmmExpr
-> CmmExpr
-> CmmExpr
-> CmmExpr
-> CmmExpr
-> Alignment
-> FCode ()
copy
  where
    -- The only time the memory might overlap is when the two arrays
    -- we were provided are the same array!
    -- TODO: Optimize branch for common case of no aliasing.
    copy :: CmmExpr
-> CmmExpr
-> CmmExpr
-> CmmExpr
-> CmmExpr
-> Alignment
-> FCode ()
copy CmmExpr
src CmmExpr
dst CmmExpr
dst_p CmmExpr
src_p CmmExpr
bytes Alignment
align = do
        DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        (CmmAGraph
moveCall, CmmAGraph
cpyCall) <- FCode CmmAGraph -> FCode CmmAGraph -> FCode (CmmAGraph, CmmAGraph)
forall a. FCode a -> FCode a -> FCode (a, a)
forkAltPair
            (FCode () -> FCode CmmAGraph
forall a. FCode a -> FCode CmmAGraph
getCode (FCode () -> FCode CmmAGraph) -> FCode () -> FCode CmmAGraph
forall a b. (a -> b) -> a -> b
$ CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemmoveCall CmmExpr
dst_p CmmExpr
src_p CmmExpr
bytes Alignment
align)
            (FCode () -> FCode CmmAGraph
forall a. FCode a -> FCode CmmAGraph
getCode (FCode () -> FCode CmmAGraph) -> FCode () -> FCode CmmAGraph
forall a b. (a -> b) -> a -> b
$ CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall  CmmExpr
dst_p CmmExpr
src_p CmmExpr
bytes Alignment
align)
        CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> FCode CmmAGraph -> FCode ()
forall (m :: * -> *) a b. Monad m => (a -> m b) -> m a -> m b
=<< CmmExpr -> CmmAGraph -> CmmAGraph -> FCode CmmAGraph
mkCmmIfThenElse (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmEqWord DynFlags
dflags CmmExpr
src CmmExpr
dst) CmmAGraph
moveCall CmmAGraph
cpyCall

emitCopyByteArray :: (CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr
                      -> Alignment -> FCode ())
                  -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr
                  -> FCode ()
emitCopyByteArray :: (CmmExpr
 -> CmmExpr
 -> CmmExpr
 -> CmmExpr
 -> CmmExpr
 -> Alignment
 -> FCode ())
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
emitCopyByteArray CmmExpr
-> CmmExpr
-> CmmExpr
-> CmmExpr
-> CmmExpr
-> Alignment
-> FCode ()
copy CmmExpr
src CmmExpr
src_off CmmExpr
dst CmmExpr
dst_off CmmExpr
n = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    let byteArrayAlignment :: Alignment
byteArrayAlignment = DynFlags -> Alignment
wordAlignment DynFlags
dflags
        srcOffAlignment :: Alignment
srcOffAlignment = CmmExpr -> Alignment
cmmExprAlignment CmmExpr
src_off
        dstOffAlignment :: Alignment
dstOffAlignment = CmmExpr -> Alignment
cmmExprAlignment CmmExpr
dst_off
        align :: Alignment
align = [Alignment] -> Alignment
forall (t :: * -> *) a. (Foldable t, Ord a) => t a -> a
minimum [Alignment
byteArrayAlignment, Alignment
srcOffAlignment, Alignment
dstOffAlignment]
    CmmExpr
dst_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExpr DynFlags
dflags (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
dst (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)) CmmExpr
dst_off
    CmmExpr
src_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExpr DynFlags
dflags (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
src (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)) CmmExpr
src_off
    CmmExpr
-> CmmExpr
-> CmmExpr
-> CmmExpr
-> CmmExpr
-> Alignment
-> FCode ()
copy CmmExpr
src CmmExpr
dst CmmExpr
dst_p CmmExpr
src_p CmmExpr
n Alignment
align

-- | Takes a source 'ByteArray#', an offset in the source array, a
-- destination 'Addr#', and the number of bytes to copy.  Copies the given
-- number of bytes from the source array to the destination memory region.
doCopyByteArrayToAddrOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyByteArrayToAddrOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyByteArrayToAddrOp CmmExpr
src CmmExpr
src_off CmmExpr
dst_p CmmExpr
bytes = do
    -- Use memcpy (we are allowed to assume the arrays aren't overlapping)
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    CmmExpr
src_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExpr DynFlags
dflags (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
src (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)) CmmExpr
src_off
    CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall CmmExpr
dst_p CmmExpr
src_p CmmExpr
bytes (Int -> Alignment
mkAlignment Int
1)

-- | Takes a source 'MutableByteArray#', an offset in the source array, a
-- destination 'Addr#', and the number of bytes to copy.  Copies the given
-- number of bytes from the source array to the destination memory region.
doCopyMutableByteArrayToAddrOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr
                               -> FCode ()
doCopyMutableByteArrayToAddrOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyMutableByteArrayToAddrOp = CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyByteArrayToAddrOp

-- | Takes a source 'Addr#', a destination 'MutableByteArray#', an offset into
-- the destination array, and the number of bytes to copy.  Copies the given
-- number of bytes from the source memory region to the destination array.
doCopyAddrToByteArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyAddrToByteArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doCopyAddrToByteArrayOp CmmExpr
src_p CmmExpr
dst CmmExpr
dst_off CmmExpr
bytes = do
    -- Use memcpy (we are allowed to assume the arrays aren't overlapping)
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    CmmExpr
dst_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExpr DynFlags
dflags (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
dst (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)) CmmExpr
dst_off
    CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall CmmExpr
dst_p CmmExpr
src_p CmmExpr
bytes (Int -> Alignment
mkAlignment Int
1)


-- ----------------------------------------------------------------------------
-- Setting byte arrays

-- | Takes a 'MutableByteArray#', an offset into the array, a length,
-- and a byte, and sets each of the selected bytes in the array to the
-- character.
doSetByteArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr
                 -> FCode ()
doSetByteArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doSetByteArrayOp CmmExpr
ba CmmExpr
off CmmExpr
len CmmExpr
c = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags

    let byteArrayAlignment :: Alignment
byteArrayAlignment = DynFlags -> Alignment
wordAlignment DynFlags
dflags -- known since BA is allocated on heap
        offsetAlignment :: Alignment
offsetAlignment = CmmExpr -> Alignment
cmmExprAlignment CmmExpr
off
        align :: Alignment
align = Alignment -> Alignment -> Alignment
forall a. Ord a => a -> a -> a
min Alignment
byteArrayAlignment Alignment
offsetAlignment

    CmmExpr
p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExpr DynFlags
dflags (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
ba (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)) CmmExpr
off
    CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemsetCall CmmExpr
p CmmExpr
c CmmExpr
len Alignment
align

-- ----------------------------------------------------------------------------
-- Allocating arrays

-- | Allocate a new array.
doNewArrayOp :: CmmFormal             -- ^ return register
             -> SMRep                 -- ^ representation of the array
             -> CLabel                -- ^ info pointer
             -> [(CmmExpr, ByteOff)]  -- ^ header payload
             -> WordOff               -- ^ array size
             -> CmmExpr               -- ^ initial element
             -> FCode ()
doNewArrayOp :: LocalReg
-> SMRep
-> CLabel
-> [(CmmExpr, Int)]
-> Int
-> CmmExpr
-> FCode ()
doNewArrayOp LocalReg
res_r SMRep
rep CLabel
info [(CmmExpr, Int)]
payload Int
n CmmExpr
init = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags

    let info_ptr :: CmmExpr
info_ptr = CLabel -> CmmExpr
mkLblExpr CLabel
info

    CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
tickyAllocPrim (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> SMRep -> Int
hdrSize DynFlags
dflags SMRep
rep))
        (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> SMRep -> Int
nonHdrSize DynFlags
dflags SMRep
rep))
        (DynFlags -> CmmExpr
zeroExpr DynFlags
dflags)

    CmmExpr
base <- SMRep -> CmmExpr -> CmmExpr -> [(CmmExpr, Int)] -> FCode CmmExpr
allocHeapClosure SMRep
rep CmmExpr
info_ptr CmmExpr
cccsExpr [(CmmExpr, Int)]
payload

    CmmReg
arr <- LocalReg -> CmmReg
CmmLocal (LocalReg -> CmmReg) -> FCode LocalReg -> FCode CmmReg
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
`fmap` CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp (DynFlags -> CmmType
bWord DynFlags
dflags)
    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign CmmReg
arr CmmExpr
base

    -- Initialise all elements of the array
    let mkOff :: Int -> CmmExpr
mkOff Int
off = DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetW DynFlags
dflags (CmmReg -> CmmExpr
CmmReg CmmReg
arr) (DynFlags -> SMRep -> Int
hdrSizeW DynFlags
dflags SMRep
rep Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
off)
        initialization :: [CmmAGraph]
initialization = [ CmmExpr -> CmmExpr -> CmmAGraph
mkStore (Int -> CmmExpr
mkOff Int
off) CmmExpr
init | Int
off <- [Int
0.. Int
n Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1] ]
    CmmAGraph -> FCode ()
emit ([CmmAGraph] -> CmmAGraph
catAGraphs [CmmAGraph]
initialization)

    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) (CmmReg -> CmmExpr
CmmReg CmmReg
arr)

-- ----------------------------------------------------------------------------
-- Copying pointer arrays

-- EZY: This code has an unusually high amount of assignTemp calls, seen
-- nowhere else in the code generator.  This is mostly because these
-- "primitive" ops result in a surprisingly large amount of code.  It
-- will likely be worthwhile to optimize what is emitted here, so that
-- our optimization passes don't waste time repeatedly optimizing the
-- same bits of code.

-- More closely imitates 'assignTemp' from the old code generator, which
-- returns a CmmExpr rather than a LocalReg.
assignTempE :: CmmExpr -> FCode CmmExpr
assignTempE :: CmmExpr -> FCode CmmExpr
assignTempE CmmExpr
e = do
    LocalReg
t <- CmmExpr -> FCode LocalReg
assignTemp CmmExpr
e
    CmmExpr -> FCode CmmExpr
forall (m :: * -> *) a. Monad m => a -> m a
return (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
t))

-- | Takes a source 'Array#', an offset in the source array, a
-- destination 'MutableArray#', an offset into the destination array,
-- and the number of elements to copy.  Copies the given number of
-- elements from the source array to the destination array.
doCopyArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> WordOff
              -> FCode ()
doCopyArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
doCopyArrayOp = (CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ())
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCopyArray CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
forall p p. p -> p -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy
  where
    -- Copy data (we assume the arrays aren't overlapping since
    -- they're of different types)
    copy :: p -> p -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy p
_src p
_dst CmmExpr
dst_p CmmExpr
src_p Int
bytes =
        do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
           CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall CmmExpr
dst_p CmmExpr
src_p (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
bytes)
               (DynFlags -> Alignment
wordAlignment DynFlags
dflags)


-- | Takes a source 'MutableArray#', an offset in the source array, a
-- destination 'MutableArray#', an offset into the destination array,
-- and the number of elements to copy.  Copies the given number of
-- elements from the source array to the destination array.
doCopyMutableArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> WordOff
                     -> FCode ()
doCopyMutableArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
doCopyMutableArrayOp = (CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ())
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCopyArray CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy
  where
    -- The only time the memory might overlap is when the two arrays
    -- we were provided are the same array!
    -- TODO: Optimize branch for common case of no aliasing.
    copy :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy CmmExpr
src CmmExpr
dst CmmExpr
dst_p CmmExpr
src_p Int
bytes = do
        DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        (CmmAGraph
moveCall, CmmAGraph
cpyCall) <- FCode CmmAGraph -> FCode CmmAGraph -> FCode (CmmAGraph, CmmAGraph)
forall a. FCode a -> FCode a -> FCode (a, a)
forkAltPair
            (FCode () -> FCode CmmAGraph
forall a. FCode a -> FCode CmmAGraph
getCode (FCode () -> FCode CmmAGraph) -> FCode () -> FCode CmmAGraph
forall a b. (a -> b) -> a -> b
$ CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemmoveCall CmmExpr
dst_p CmmExpr
src_p (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
bytes)
             (DynFlags -> Alignment
wordAlignment DynFlags
dflags))
            (FCode () -> FCode CmmAGraph
forall a. FCode a -> FCode CmmAGraph
getCode (FCode () -> FCode CmmAGraph) -> FCode () -> FCode CmmAGraph
forall a b. (a -> b) -> a -> b
$ CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall  CmmExpr
dst_p CmmExpr
src_p (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
bytes)
             (DynFlags -> Alignment
wordAlignment DynFlags
dflags))
        CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> FCode CmmAGraph -> FCode ()
forall (m :: * -> *) a b. Monad m => (a -> m b) -> m a -> m b
=<< CmmExpr -> CmmAGraph -> CmmAGraph -> FCode CmmAGraph
mkCmmIfThenElse (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmEqWord DynFlags
dflags CmmExpr
src CmmExpr
dst) CmmAGraph
moveCall CmmAGraph
cpyCall

emitCopyArray :: (CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> ByteOff
                  -> FCode ())  -- ^ copy function
              -> CmmExpr        -- ^ source array
              -> CmmExpr        -- ^ offset in source array
              -> CmmExpr        -- ^ destination array
              -> CmmExpr        -- ^ offset in destination array
              -> WordOff        -- ^ number of elements to copy
              -> FCode ()
emitCopyArray :: (CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ())
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCopyArray CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy CmmExpr
src0 CmmExpr
src_off CmmExpr
dst0 CmmExpr
dst_off0 Int
n =
    Bool -> FCode () -> FCode ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (Int
n Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
/= Int
0) (FCode () -> FCode ()) -> FCode () -> FCode ()
forall a b. (a -> b) -> a -> b
$ do
        DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags

        -- Passed as arguments (be careful)
        CmmExpr
src     <- CmmExpr -> FCode CmmExpr
assignTempE CmmExpr
src0
        CmmExpr
dst     <- CmmExpr -> FCode CmmExpr
assignTempE CmmExpr
dst0
        CmmExpr
dst_off <- CmmExpr -> FCode CmmExpr
assignTempE CmmExpr
dst_off0

        -- Nonmoving collector write barrier
        DynFlags -> Int -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCopyUpdRemSetPush DynFlags
dflags (DynFlags -> Int
arrPtrsHdrSizeW DynFlags
dflags) CmmExpr
dst CmmExpr
dst_off Int
n

        -- Set the dirty bit in the header.
        CmmAGraph -> FCode ()
emit (CmmExpr -> CmmExpr -> CmmAGraph
setInfo CmmExpr
dst (CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel CLabel
mkMAP_DIRTY_infoLabel)))

        CmmExpr
dst_elems_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
dst
                       (DynFlags -> Int
arrPtrsHdrSize DynFlags
dflags)
        CmmExpr
dst_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExprW DynFlags
dflags CmmExpr
dst_elems_p CmmExpr
dst_off
        CmmExpr
src_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExprW DynFlags
dflags
                 (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
src (DynFlags -> Int
arrPtrsHdrSize DynFlags
dflags)) CmmExpr
src_off
        let bytes :: Int
bytes = DynFlags -> Int -> Int
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags Int
n

        CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy CmmExpr
src CmmExpr
dst CmmExpr
dst_p CmmExpr
src_p Int
bytes

        -- The base address of the destination card table
        CmmExpr
dst_cards_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExprW DynFlags
dflags CmmExpr
dst_elems_p
                       (DynFlags -> CmmExpr -> CmmExpr
loadArrPtrsSize DynFlags
dflags CmmExpr
dst)

        CmmExpr -> CmmExpr -> Int -> FCode ()
emitSetCards CmmExpr
dst_off CmmExpr
dst_cards_p Int
n

doCopySmallArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> WordOff
                   -> FCode ()
doCopySmallArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
doCopySmallArrayOp = (CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ())
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCopySmallArray CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
forall p p. p -> p -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy
  where
    -- Copy data (we assume the arrays aren't overlapping since
    -- they're of different types)
    copy :: p -> p -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy p
_src p
_dst CmmExpr
dst_p CmmExpr
src_p Int
bytes =
        do DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
           CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall CmmExpr
dst_p CmmExpr
src_p (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
bytes)
               (DynFlags -> Alignment
wordAlignment DynFlags
dflags)


doCopySmallMutableArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> WordOff
                          -> FCode ()
doCopySmallMutableArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
doCopySmallMutableArrayOp = (CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ())
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCopySmallArray CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy
  where
    -- The only time the memory might overlap is when the two arrays
    -- we were provided are the same array!
    -- TODO: Optimize branch for common case of no aliasing.
    copy :: CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy CmmExpr
src CmmExpr
dst CmmExpr
dst_p CmmExpr
src_p Int
bytes = do
        DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
        (CmmAGraph
moveCall, CmmAGraph
cpyCall) <- FCode CmmAGraph -> FCode CmmAGraph -> FCode (CmmAGraph, CmmAGraph)
forall a. FCode a -> FCode a -> FCode (a, a)
forkAltPair
            (FCode () -> FCode CmmAGraph
forall a. FCode a -> FCode CmmAGraph
getCode (FCode () -> FCode CmmAGraph) -> FCode () -> FCode CmmAGraph
forall a b. (a -> b) -> a -> b
$ CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemmoveCall CmmExpr
dst_p CmmExpr
src_p (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
bytes)
             (DynFlags -> Alignment
wordAlignment DynFlags
dflags))
            (FCode () -> FCode CmmAGraph
forall a. FCode a -> FCode CmmAGraph
getCode (FCode () -> FCode CmmAGraph) -> FCode () -> FCode CmmAGraph
forall a b. (a -> b) -> a -> b
$ CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall  CmmExpr
dst_p CmmExpr
src_p (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
bytes)
             (DynFlags -> Alignment
wordAlignment DynFlags
dflags))
        CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> FCode CmmAGraph -> FCode ()
forall (m :: * -> *) a b. Monad m => (a -> m b) -> m a -> m b
=<< CmmExpr -> CmmAGraph -> CmmAGraph -> FCode CmmAGraph
mkCmmIfThenElse (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmEqWord DynFlags
dflags CmmExpr
src CmmExpr
dst) CmmAGraph
moveCall CmmAGraph
cpyCall

emitCopySmallArray :: (CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> ByteOff
                       -> FCode ())  -- ^ copy function
                   -> CmmExpr        -- ^ source array
                   -> CmmExpr        -- ^ offset in source array
                   -> CmmExpr        -- ^ destination array
                   -> CmmExpr        -- ^ offset in destination array
                   -> WordOff        -- ^ number of elements to copy
                   -> FCode ()
emitCopySmallArray :: (CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ())
-> CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCopySmallArray CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy CmmExpr
src0 CmmExpr
src_off CmmExpr
dst0 CmmExpr
dst_off Int
n =
    Bool -> FCode () -> FCode ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (Int
n Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
/= Int
0) (FCode () -> FCode ()) -> FCode () -> FCode ()
forall a b. (a -> b) -> a -> b
$ do
        DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags

        -- Passed as arguments (be careful)
        CmmExpr
src     <- CmmExpr -> FCode CmmExpr
assignTempE CmmExpr
src0
        CmmExpr
dst     <- CmmExpr -> FCode CmmExpr
assignTempE CmmExpr
dst0

        -- Nonmoving collector write barrier
        DynFlags -> Int -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCopyUpdRemSetPush DynFlags
dflags (DynFlags -> Int
smallArrPtrsHdrSizeW DynFlags
dflags) CmmExpr
dst CmmExpr
dst_off Int
n

        -- Set the dirty bit in the header.
        CmmAGraph -> FCode ()
emit (CmmExpr -> CmmExpr -> CmmAGraph
setInfo CmmExpr
dst (CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel CLabel
mkSMAP_DIRTY_infoLabel)))

        CmmExpr
dst_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExprW DynFlags
dflags
                 (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
dst (DynFlags -> Int
smallArrPtrsHdrSize DynFlags
dflags)) CmmExpr
dst_off
        CmmExpr
src_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExprW DynFlags
dflags
                 (DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags CmmExpr
src (DynFlags -> Int
smallArrPtrsHdrSize DynFlags
dflags)) CmmExpr
src_off
        let bytes :: Int
bytes = DynFlags -> Int -> Int
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags Int
n

        CmmExpr -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
copy CmmExpr
src CmmExpr
dst CmmExpr
dst_p CmmExpr
src_p Int
bytes

-- | Takes an info table label, a register to return the newly
-- allocated array in, a source array, an offset in the source array,
-- and the number of elements to copy. Allocates a new array and
-- initializes it from the source array.
emitCloneArray :: CLabel -> CmmFormal -> CmmExpr -> CmmExpr -> WordOff
               -> FCode ()
emitCloneArray :: CLabel -> LocalReg -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCloneArray CLabel
info_p LocalReg
res_r CmmExpr
src CmmExpr
src_off Int
n = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags

    let info_ptr :: CmmExpr
info_ptr = CLabel -> CmmExpr
mkLblExpr CLabel
info_p
        rep :: SMRep
rep = DynFlags -> Int -> SMRep
arrPtrsRep DynFlags
dflags Int
n

    CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
tickyAllocPrim (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
arrPtrsHdrSize DynFlags
dflags))
        (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> SMRep -> Int
nonHdrSize DynFlags
dflags SMRep
rep))
        (DynFlags -> CmmExpr
zeroExpr DynFlags
dflags)

    let hdr_size :: Int
hdr_size = DynFlags -> Int
fixedHdrSize DynFlags
dflags

    CmmExpr
base <- SMRep -> CmmExpr -> CmmExpr -> [(CmmExpr, Int)] -> FCode CmmExpr
allocHeapClosure SMRep
rep CmmExpr
info_ptr CmmExpr
cccsExpr
                     [ (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
n,
                        Int
hdr_size Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DynFlags -> Int
oFFSET_StgMutArrPtrs_ptrs DynFlags
dflags)
                     , (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (SMRep -> Int
nonHdrSizeW SMRep
rep),
                        Int
hdr_size Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DynFlags -> Int
oFFSET_StgMutArrPtrs_size DynFlags
dflags)
                     ]

    CmmReg
arr <- LocalReg -> CmmReg
CmmLocal (LocalReg -> CmmReg) -> FCode LocalReg -> FCode CmmReg
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
`fmap` CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp (DynFlags -> CmmType
bWord DynFlags
dflags)
    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign CmmReg
arr CmmExpr
base

    CmmExpr
dst_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags (CmmReg -> CmmExpr
CmmReg CmmReg
arr)
             (DynFlags -> Int
arrPtrsHdrSize DynFlags
dflags)
    CmmExpr
src_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExprW DynFlags
dflags CmmExpr
src
             (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmAddWord DynFlags
dflags
              (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
arrPtrsHdrSizeW DynFlags
dflags)) CmmExpr
src_off)

    CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall CmmExpr
dst_p CmmExpr
src_p (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int -> Int
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags Int
n))
        (DynFlags -> Alignment
wordAlignment DynFlags
dflags)

    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) (CmmReg -> CmmExpr
CmmReg CmmReg
arr)

-- | Takes an info table label, a register to return the newly
-- allocated array in, a source array, an offset in the source array,
-- and the number of elements to copy. Allocates a new array and
-- initializes it from the source array.
emitCloneSmallArray :: CLabel -> CmmFormal -> CmmExpr -> CmmExpr -> WordOff
                    -> FCode ()
emitCloneSmallArray :: CLabel -> LocalReg -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCloneSmallArray CLabel
info_p LocalReg
res_r CmmExpr
src CmmExpr
src_off Int
n = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags

    let info_ptr :: CmmExpr
info_ptr = CLabel -> CmmExpr
mkLblExpr CLabel
info_p
        rep :: SMRep
rep = Int -> SMRep
smallArrPtrsRep Int
n

    CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
tickyAllocPrim (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
smallArrPtrsHdrSize DynFlags
dflags))
        (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> SMRep -> Int
nonHdrSize DynFlags
dflags SMRep
rep))
        (DynFlags -> CmmExpr
zeroExpr DynFlags
dflags)

    let hdr_size :: Int
hdr_size = DynFlags -> Int
fixedHdrSize DynFlags
dflags

    CmmExpr
base <- SMRep -> CmmExpr -> CmmExpr -> [(CmmExpr, Int)] -> FCode CmmExpr
allocHeapClosure SMRep
rep CmmExpr
info_ptr CmmExpr
cccsExpr
                     [ (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
n,
                        Int
hdr_size Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DynFlags -> Int
oFFSET_StgSmallMutArrPtrs_ptrs DynFlags
dflags)
                     ]

    CmmReg
arr <- LocalReg -> CmmReg
CmmLocal (LocalReg -> CmmReg) -> FCode LocalReg -> FCode CmmReg
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
`fmap` CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp (DynFlags -> CmmType
bWord DynFlags
dflags)
    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign CmmReg
arr CmmExpr
base

    CmmExpr
dst_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> Int -> CmmExpr
cmmOffsetB DynFlags
dflags (CmmReg -> CmmExpr
CmmReg CmmReg
arr)
             (DynFlags -> Int
smallArrPtrsHdrSize DynFlags
dflags)
    CmmExpr
src_p <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmOffsetExprW DynFlags
dflags CmmExpr
src
             (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmAddWord DynFlags
dflags
              (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
smallArrPtrsHdrSizeW DynFlags
dflags)) CmmExpr
src_off)

    CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall CmmExpr
dst_p CmmExpr
src_p (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int -> Int
forall a. Num a => DynFlags -> a -> a
wordsToBytes DynFlags
dflags Int
n))
        (DynFlags -> Alignment
wordAlignment DynFlags
dflags)

    CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res_r) (CmmReg -> CmmExpr
CmmReg CmmReg
arr)

-- | Takes and offset in the destination array, the base address of
-- the card table, and the number of elements affected (*not* the
-- number of cards). The number of elements may not be zero.
-- Marks the relevant cards as dirty.
emitSetCards :: CmmExpr -> CmmExpr -> WordOff -> FCode ()
emitSetCards :: CmmExpr -> CmmExpr -> Int -> FCode ()
emitSetCards CmmExpr
dst_start CmmExpr
dst_cards_start Int
n = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    CmmExpr
start_card <- CmmExpr -> FCode CmmExpr
assignTempE (CmmExpr -> FCode CmmExpr) -> CmmExpr -> FCode CmmExpr
forall a b. (a -> b) -> a -> b
$ DynFlags -> CmmExpr -> CmmExpr
cardCmm DynFlags
dflags CmmExpr
dst_start
    let end_card :: CmmExpr
end_card = DynFlags -> CmmExpr -> CmmExpr
cardCmm DynFlags
dflags
                   (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmSubWord DynFlags
dflags
                    (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmAddWord DynFlags
dflags CmmExpr
dst_start (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
n))
                    (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
1))
    CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemsetCall (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmAddWord DynFlags
dflags CmmExpr
dst_cards_start CmmExpr
start_card)
        (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
1)
        (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmAddWord DynFlags
dflags (DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmSubWord DynFlags
dflags CmmExpr
end_card CmmExpr
start_card) (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
1))
        (Int -> Alignment
mkAlignment Int
1) -- no alignment (1 byte)

-- Convert an element index to a card index
cardCmm :: DynFlags -> CmmExpr -> CmmExpr
cardCmm :: DynFlags -> CmmExpr -> CmmExpr
cardCmm DynFlags
dflags CmmExpr
i =
    DynFlags -> CmmExpr -> CmmExpr -> CmmExpr
cmmUShrWord DynFlags
dflags CmmExpr
i (DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags (DynFlags -> Int
mUT_ARR_PTRS_CARD_BITS DynFlags
dflags))

------------------------------------------------------------------------------
-- SmallArray PrimOp implementations

doReadSmallPtrArrayOp :: LocalReg
                      -> CmmExpr
                      -> CmmExpr
                      -> FCode ()
doReadSmallPtrArrayOp :: LocalReg -> CmmExpr -> CmmExpr -> FCode ()
doReadSmallPtrArrayOp LocalReg
res CmmExpr
addr CmmExpr
idx = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    Int
-> Maybe MachOp
-> CmmType
-> LocalReg
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
mkBasicIndexedRead (DynFlags -> Int
smallArrPtrsHdrSize DynFlags
dflags) Maybe MachOp
forall a. Maybe a
Nothing (DynFlags -> CmmType
gcWord DynFlags
dflags) LocalReg
res CmmExpr
addr
        (DynFlags -> CmmType
gcWord DynFlags
dflags) CmmExpr
idx

doWriteSmallPtrArrayOp :: CmmExpr
                       -> CmmExpr
                       -> CmmExpr
                       -> FCode ()
doWriteSmallPtrArrayOp :: CmmExpr -> CmmExpr -> CmmExpr -> FCode ()
doWriteSmallPtrArrayOp CmmExpr
addr CmmExpr
idx CmmExpr
val = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    let ty :: CmmType
ty = DynFlags -> CmmExpr -> CmmType
cmmExprType DynFlags
dflags CmmExpr
val

    -- Update remembered set for non-moving collector
    LocalReg
tmp <- CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
ty
    Int
-> Maybe MachOp
-> CmmType
-> LocalReg
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
mkBasicIndexedRead (DynFlags -> Int
smallArrPtrsHdrSize DynFlags
dflags) Maybe MachOp
forall a. Maybe a
Nothing CmmType
ty LocalReg
tmp CmmExpr
addr CmmType
ty CmmExpr
idx
    DynFlags -> FCode () -> FCode ()
forall a. DynFlags -> FCode a -> FCode ()
whenUpdRemSetEnabled DynFlags
dflags (FCode () -> FCode ()) -> FCode () -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmExpr -> FCode ()
emitUpdRemSetPush (CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
tmp))

    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall [] CallishMachOp
MO_WriteBarrier [] -- #12469
    Int
-> Maybe MachOp
-> CmmExpr
-> CmmType
-> CmmExpr
-> CmmExpr
-> FCode ()
mkBasicIndexedWrite (DynFlags -> Int
smallArrPtrsHdrSize DynFlags
dflags) Maybe MachOp
forall a. Maybe a
Nothing CmmExpr
addr CmmType
ty CmmExpr
idx CmmExpr
val
    CmmAGraph -> FCode ()
emit (CmmExpr -> CmmExpr -> CmmAGraph
setInfo CmmExpr
addr (CmmLit -> CmmExpr
CmmLit (CLabel -> CmmLit
CmmLabel CLabel
mkSMAP_DIRTY_infoLabel)))

------------------------------------------------------------------------------
-- Atomic read-modify-write

-- | Emit an atomic modification to a byte array element. The result
-- reg contains that previous value of the element. Implies a full
-- memory barrier.
doAtomicRMW :: LocalReg      -- ^ Result reg
            -> AtomicMachOp  -- ^ Atomic op (e.g. add)
            -> CmmExpr       -- ^ MutableByteArray#
            -> CmmExpr       -- ^ Index
            -> CmmType       -- ^ Type of element by which we are indexing
            -> CmmExpr       -- ^ Op argument (e.g. amount to add)
            -> FCode ()
doAtomicRMW :: LocalReg
-> AtomicMachOp
-> CmmExpr
-> CmmExpr
-> CmmType
-> CmmExpr
-> FCode ()
doAtomicRMW LocalReg
res AtomicMachOp
amop CmmExpr
mba CmmExpr
idx CmmType
idx_ty CmmExpr
n = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    let width :: Width
width = CmmType -> Width
typeWidth CmmType
idx_ty
        addr :: CmmExpr
addr  = DynFlags -> Int -> Width -> CmmExpr -> CmmExpr -> CmmExpr
cmmIndexOffExpr DynFlags
dflags (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)
                Width
width CmmExpr
mba CmmExpr
idx
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
res ]
        (Width -> AtomicMachOp -> CallishMachOp
MO_AtomicRMW Width
width AtomicMachOp
amop)
        [ CmmExpr
addr, CmmExpr
n ]

-- | Emit an atomic read to a byte array that acts as a memory barrier.
doAtomicReadByteArray
    :: LocalReg  -- ^ Result reg
    -> CmmExpr   -- ^ MutableByteArray#
    -> CmmExpr   -- ^ Index
    -> CmmType   -- ^ Type of element by which we are indexing
    -> FCode ()
doAtomicReadByteArray :: LocalReg -> CmmExpr -> CmmExpr -> CmmType -> FCode ()
doAtomicReadByteArray LocalReg
res CmmExpr
mba CmmExpr
idx CmmType
idx_ty = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    let width :: Width
width = CmmType -> Width
typeWidth CmmType
idx_ty
        addr :: CmmExpr
addr  = DynFlags -> Int -> Width -> CmmExpr -> CmmExpr -> CmmExpr
cmmIndexOffExpr DynFlags
dflags (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)
                Width
width CmmExpr
mba CmmExpr
idx
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
res ]
        (Width -> CallishMachOp
MO_AtomicRead Width
width)
        [ CmmExpr
addr ]

-- | Emit an atomic write to a byte array that acts as a memory barrier.
doAtomicWriteByteArray
    :: CmmExpr   -- ^ MutableByteArray#
    -> CmmExpr   -- ^ Index
    -> CmmType   -- ^ Type of element by which we are indexing
    -> CmmExpr   -- ^ Value to write
    -> FCode ()
doAtomicWriteByteArray :: CmmExpr -> CmmExpr -> CmmType -> CmmExpr -> FCode ()
doAtomicWriteByteArray CmmExpr
mba CmmExpr
idx CmmType
idx_ty CmmExpr
val = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    let width :: Width
width = CmmType -> Width
typeWidth CmmType
idx_ty
        addr :: CmmExpr
addr  = DynFlags -> Int -> Width -> CmmExpr -> CmmExpr -> CmmExpr
cmmIndexOffExpr DynFlags
dflags (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)
                Width
width CmmExpr
mba CmmExpr
idx
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ {- no results -} ]
        (Width -> CallishMachOp
MO_AtomicWrite Width
width)
        [ CmmExpr
addr, CmmExpr
val ]

doCasByteArray
    :: LocalReg  -- ^ Result reg
    -> CmmExpr   -- ^ MutableByteArray#
    -> CmmExpr   -- ^ Index
    -> CmmType   -- ^ Type of element by which we are indexing
    -> CmmExpr   -- ^ Old value
    -> CmmExpr   -- ^ New value
    -> FCode ()
doCasByteArray :: LocalReg
-> CmmExpr -> CmmExpr -> CmmType -> CmmExpr -> CmmExpr -> FCode ()
doCasByteArray LocalReg
res CmmExpr
mba CmmExpr
idx CmmType
idx_ty CmmExpr
old CmmExpr
new = do
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    let width :: Width
width = (CmmType -> Width
typeWidth CmmType
idx_ty)
        addr :: CmmExpr
addr = DynFlags -> Int -> Width -> CmmExpr -> CmmExpr -> CmmExpr
cmmIndexOffExpr DynFlags
dflags (DynFlags -> Int
arrWordsHdrSize DynFlags
dflags)
               Width
width CmmExpr
mba CmmExpr
idx
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
res ]
        (Width -> CallishMachOp
MO_Cmpxchg Width
width)
        [ CmmExpr
addr, CmmExpr
old, CmmExpr
new ]

------------------------------------------------------------------------------
-- Helpers for emitting function calls

-- | Emit a call to @memcpy@.
emitMemcpyCall :: CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall :: CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemcpyCall CmmExpr
dst CmmExpr
src CmmExpr
n Alignment
align = do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ {-no results-} ]
        (Int -> CallishMachOp
MO_Memcpy (Alignment -> Int
alignmentBytes Alignment
align))
        [ CmmExpr
dst, CmmExpr
src, CmmExpr
n ]

-- | Emit a call to @memmove@.
emitMemmoveCall :: CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemmoveCall :: CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemmoveCall CmmExpr
dst CmmExpr
src CmmExpr
n Alignment
align = do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ {- no results -} ]
        (Int -> CallishMachOp
MO_Memmove (Alignment -> Int
alignmentBytes Alignment
align))
        [ CmmExpr
dst, CmmExpr
src, CmmExpr
n ]

-- | Emit a call to @memset@.  The second argument must fit inside an
-- unsigned char.
emitMemsetCall :: CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemsetCall :: CmmExpr -> CmmExpr -> CmmExpr -> Alignment -> FCode ()
emitMemsetCall CmmExpr
dst CmmExpr
c CmmExpr
n Alignment
align = do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ {- no results -} ]
        (Int -> CallishMachOp
MO_Memset (Alignment -> Int
alignmentBytes Alignment
align))
        [ CmmExpr
dst, CmmExpr
c, CmmExpr
n ]

emitMemcmpCall :: LocalReg -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitMemcmpCall :: LocalReg -> CmmExpr -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitMemcmpCall LocalReg
res CmmExpr
ptr1 CmmExpr
ptr2 CmmExpr
n Int
align = do
    -- 'MO_Memcmp' is assumed to return an 32bit 'CInt' because all
    -- code-gens currently call out to the @memcmp(3)@ C function.
    -- This was easier than moving the sign-extensions into
    -- all the code-gens.
    DynFlags
dflags <- FCode DynFlags
forall (m :: * -> *). HasDynFlags m => m DynFlags
getDynFlags
    let is32Bit :: Bool
is32Bit = CmmType -> Width
typeWidth (LocalReg -> CmmType
localRegType LocalReg
res) Width -> Width -> Bool
forall a. Eq a => a -> a -> Bool
== Width
W32

    LocalReg
cres <- if Bool
is32Bit
              then LocalReg -> FCode LocalReg
forall (m :: * -> *) a. Monad m => a -> m a
return LocalReg
res
              else CmmType -> FCode LocalReg
forall (m :: * -> *). MonadUnique m => CmmType -> m LocalReg
newTemp CmmType
b32

    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
cres ]
        (Int -> CallishMachOp
MO_Memcmp Int
align)
        [ CmmExpr
ptr1, CmmExpr
ptr2, CmmExpr
n ]

    Bool -> FCode () -> FCode ()
forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
unless Bool
is32Bit (FCode () -> FCode ()) -> FCode () -> FCode ()
forall a b. (a -> b) -> a -> b
$ do
      CmmAGraph -> FCode ()
emit (CmmAGraph -> FCode ()) -> CmmAGraph -> FCode ()
forall a b. (a -> b) -> a -> b
$ CmmReg -> CmmExpr -> CmmAGraph
mkAssign (LocalReg -> CmmReg
CmmLocal LocalReg
res)
                      (MachOp -> [CmmExpr] -> CmmExpr
CmmMachOp
                         (DynFlags -> MachOp
mo_s_32ToWord DynFlags
dflags)
                         [(CmmReg -> CmmExpr
CmmReg (LocalReg -> CmmReg
CmmLocal LocalReg
cres))])

emitBSwapCall :: LocalReg -> CmmExpr -> Width -> FCode ()
emitBSwapCall :: LocalReg -> CmmExpr -> Width -> FCode ()
emitBSwapCall LocalReg
res CmmExpr
x Width
width = do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
res ]
        (Width -> CallishMachOp
MO_BSwap Width
width)
        [ CmmExpr
x ]

emitBRevCall :: LocalReg -> CmmExpr -> Width -> FCode ()
emitBRevCall :: LocalReg -> CmmExpr -> Width -> FCode ()
emitBRevCall LocalReg
res CmmExpr
x Width
width = do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
res ]
        (Width -> CallishMachOp
MO_BRev Width
width)
        [ CmmExpr
x ]

emitPopCntCall :: LocalReg -> CmmExpr -> Width -> FCode ()
emitPopCntCall :: LocalReg -> CmmExpr -> Width -> FCode ()
emitPopCntCall LocalReg
res CmmExpr
x Width
width = do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
res ]
        (Width -> CallishMachOp
MO_PopCnt Width
width)
        [ CmmExpr
x ]

emitPdepCall :: LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPdepCall :: LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPdepCall LocalReg
res CmmExpr
x CmmExpr
y Width
width = do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
res ]
        (Width -> CallishMachOp
MO_Pdep Width
width)
        [ CmmExpr
x, CmmExpr
y ]

emitPextCall :: LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPextCall :: LocalReg -> CmmExpr -> CmmExpr -> Width -> FCode ()
emitPextCall LocalReg
res CmmExpr
x CmmExpr
y Width
width = do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
res ]
        (Width -> CallishMachOp
MO_Pext Width
width)
        [ CmmExpr
x, CmmExpr
y ]

emitClzCall :: LocalReg -> CmmExpr -> Width -> FCode ()
emitClzCall :: LocalReg -> CmmExpr -> Width -> FCode ()
emitClzCall LocalReg
res CmmExpr
x Width
width = do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
res ]
        (Width -> CallishMachOp
MO_Clz Width
width)
        [ CmmExpr
x ]

emitCtzCall :: LocalReg -> CmmExpr -> Width -> FCode ()
emitCtzCall :: LocalReg -> CmmExpr -> Width -> FCode ()
emitCtzCall LocalReg
res CmmExpr
x Width
width = do
    [LocalReg] -> CallishMachOp -> [CmmExpr] -> FCode ()
emitPrimCall
        [ LocalReg
res ]
        (Width -> CallishMachOp
MO_Ctz Width
width)
        [ CmmExpr
x ]

---------------------------------------------------------------------------
-- Pushing to the update remembered set
---------------------------------------------------------------------------

-- | Push a range of pointer-array elements that are about to be copied over to
-- the update remembered set.
emitCopyUpdRemSetPush :: DynFlags
                      -> WordOff    -- ^ array header size
                      -> CmmExpr    -- ^ destination array
                      -> CmmExpr    -- ^ offset in destination array (in words)
                      -> Int        -- ^ number of elements to copy
                      -> FCode ()
emitCopyUpdRemSetPush :: DynFlags -> Int -> CmmExpr -> CmmExpr -> Int -> FCode ()
emitCopyUpdRemSetPush DynFlags
_dflags Int
_hdr_size CmmExpr
_dst CmmExpr
_dst_off Int
0 = () -> FCode ()
forall (m :: * -> *) a. Monad m => a -> m a
return ()
emitCopyUpdRemSetPush DynFlags
dflags Int
hdr_size CmmExpr
dst CmmExpr
dst_off Int
n =
    DynFlags -> FCode () -> FCode ()
forall a. DynFlags -> FCode a -> FCode ()
whenUpdRemSetEnabled DynFlags
dflags (FCode () -> FCode ()) -> FCode () -> FCode ()
forall a b. (a -> b) -> a -> b
$ do
        Int
updfr_off <- FCode Int
getUpdFrameOff
        CmmAGraph
graph <- CmmExpr
-> (Convention, Convention)
-> [LocalReg]
-> [CmmExpr]
-> Int
-> [CmmExpr]
-> FCode CmmAGraph
mkCall CmmExpr
lbl (Convention
NativeNodeCall,Convention
NativeReturn) [] [CmmExpr]
args Int
updfr_off []
        CmmAGraph -> FCode ()
emit CmmAGraph
graph
  where
    lbl :: CmmExpr
lbl = CLabel -> CmmExpr
mkLblExpr (CLabel -> CmmExpr) -> CLabel -> CmmExpr
forall a b. (a -> b) -> a -> b
$ PrimCall -> CLabel
mkPrimCallLabel
          (PrimCall -> CLabel) -> PrimCall -> CLabel
forall a b. (a -> b) -> a -> b
$ FastString -> UnitId -> PrimCall
PrimCall (String -> FastString
fsLit String
"stg_copyArray_barrier") UnitId
rtsUnitId
    args :: [CmmExpr]
args =
      [ DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
hdr_size
      , CmmExpr
dst
      , CmmExpr
dst_off
      , DynFlags -> Int -> CmmExpr
mkIntExpr DynFlags
dflags Int
n
      ]