By using the FJ.Core library, I was able to create a WriteableBitmapExtentions class that does a lot the work for encoding and decoding jpg's in Silverlight 4.  Most of the ideas came from Kodierer, but his decoder was jacked and it took me most of the day to figure out why, so I though it would be nice to put it here in case someone else needed it.

public static class WriteableBitmapExtentions
    {
        public static byte[] ToByteArray(this WriteableBitmap bmp)
        {
            int[] p = bmp.Pixels;
            int len = p.Length * 4;
            byte[] result = new byte[len]; // ARGB    
            Buffer.BlockCopy(p, 0, result, 0, len);
            return result;
        }

        public static void FromByteArray(this WriteableBitmap bmp, byte[] buffer)
        {
            Buffer.BlockCopy(buffer, 0, bmp.Pixels, 0, buffer.Length);
        }
        public static byte[] EncodeJpeg(this WriteableBitmap bmp)
        {
            MemoryStream ms = new MemoryStream();
            EncodeJpeg(bmp, ms);
            return ms.ToArray();
        }
        public static void EncodeJpeg(this WriteableBitmap bmp, Stream destinationStream)
        {
            // Init buffer in FluxJpeg format   
 
            int w = bmp.PixelWidth;
            int h = bmp.PixelHeight;
            int[] p = bmp.Pixels;
            byte[][,] pixelsForJpeg = new byte[3][,];

            // RGB colors   

            pixelsForJpeg[0] = new byte[w, h];
            pixelsForJpeg[1] = new byte[w, h];
            pixelsForJpeg[2] = new byte[w, h];

            // Copy WriteableBitmap data into buffer for FluxJpeg   
            for (int row = 0; row < h; row++)
            {
                for (int column = 0; column < w; column++)
                {
                    int pixel = bmp.Pixels[w * row + column];
                    pixelsForJpeg[0][column, row] = (byte)(pixel >> 16);
                    pixelsForJpeg[1][column, row] = (byte)(pixel >> 8);
                    pixelsForJpeg[2][column, row] = (byte)pixel;
                }
            }
            // Encode Image as JPEG using the FluxJpeg library   
            // and write to destination stream  

            ColorModel cm = new ColorModel { colorspace = ColorSpace.RGB };
            FluxJpeg.Core.Image jpegImage = new FluxJpeg.Core.Image(cm, pixelsForJpeg);
            JpegEncoder encoder = new JpegEncoder(jpegImage, 50, destinationStream);
            encoder.Encode();
        }
        public static WriteableBitmap DecodeJpeg(Stream sourceStream)
        {
            // Decode JPEG from stream   

            var decoder = new FluxJpeg.Core.Decoder.JpegDecoder(sourceStream);
            var jpegDecoded = decoder.Decode();
            var img = jpegDecoded.Image;
            img.ChangeColorSpace(ColorSpace.RGB);

            // Init Buffer 

            int w = img.Width;
            int h = img.Height;
            var result = new WriteableBitmap(w, h);
            int[] p = result.Pixels;
            byte[][,] pixelsFromJpeg = img.Raster;
            int i = 0;
              
            for (int y = 0; y < h; y++)
            {
                for (int x = 0; x < w; x++)
                {
              
                        p[i++] = (0xFF << 24)                    // A   
                            | (pixelsFromJpeg[0][x, y] << 16) // R     
                            | (pixelsFromJpeg[1][x, y] << 8)  // G      
                            | pixelsFromJpeg[2][x, y];       // B      
               
                }
            }
     
            return result;
        }
 
    }