Fix crashes

Based off the pull request at BradLarson#243

Author: matt-oakes

framework/Source/Apple/PictureInput.swift

@@ -10,18 +10,40 @@ import UIKit
 import Cocoa
 #endif
 
+public enum PictureInputError: Error, CustomStringConvertible {
+    case zeroSizedImageError
+    case dataProviderNilError
+    case noSuchImageError(imageName: String)
+
+    public var errorDescription: String {
+        switch self {
+        case .zeroSizedImageError:
+            return "Tried to pass in a zero-sized image"
+        case .dataProviderNilError:
+            return "Unable to retrieve image dataProvider"
+        case .noSuchImageError(let imageName):
+            return "No such image named: \(imageName) in your application bundle"
+        }
+    }
+
+    public var description: String {
+        return "<\(type(of: self)): errorDescription = \(self.errorDescription)>"
+    }
+}
+
 public class PictureInput: ImageSource {
     public let targets = TargetContainer()
-    var imageFramebuffer:Framebuffer!
+    var imageFramebuffer:Framebuffer?
+    public var framebufferUserInfo:[AnyHashable:Any]?
     var hasProcessedImage:Bool = false
 
-    public init(image:CGImage, smoothlyScaleOutput:Bool = false, orientation:ImageOrientation = .portrait) {
+    public init(image:CGImage, smoothlyScaleOutput:Bool = false, orientation:ImageOrientation = .portrait) throws {
         // TODO: Dispatch this whole thing asynchronously to move image loading off main thread
         let widthOfImage = GLint(image.width)
         let heightOfImage = GLint(image.height)
 
         // If passed an empty image reference, CGContextDrawImage will fail in future versions of the SDK.
-        guard((widthOfImage > 0) && (heightOfImage > 0)) else { fatalError("Tried to pass in a zero-sized image") }
+        guard((widthOfImage > 0) && (heightOfImage > 0)) else { throw PictureInputError.zeroSizedImageError }
 
         var widthToUseForTexture = widthOfImage
         var heightToUseForTexture = heightOfImage
@@ -67,12 +89,12 @@ public class PictureInput: ImageSource {
                     if (bitmapInfo.contains(.byteOrder32Little)) {
                         /* Little endian, for alpha-first we can use this bitmap directly in GL */
                         if ((alphaInfo != CGImageAlphaInfo.premultipliedFirst) && (alphaInfo != CGImageAlphaInfo.first) && (alphaInfo != CGImageAlphaInfo.noneSkipFirst)) {
-                                shouldRedrawUsingCoreGraphics = true
+                            shouldRedrawUsingCoreGraphics = true
                         }
                     } else if ((bitmapInfo.contains(CGBitmapInfo())) || (bitmapInfo.contains(.byteOrder32Big))) {
                         /* Big endian, for alpha-last we can use this bitmap directly in GL */
                         if ((alphaInfo != CGImageAlphaInfo.premultipliedLast) && (alphaInfo != CGImageAlphaInfo.last) && (alphaInfo != CGImageAlphaInfo.noneSkipLast)) {
-                                shouldRedrawUsingCoreGraphics = true
+                            shouldRedrawUsingCoreGraphics = true
                         } else {
                             /* Can access directly using GL_RGBA pixel format */
                             format = GL_RGBA
@@ -82,36 +104,30 @@ public class PictureInput: ImageSource {
             }
         }
 
-        //    CFAbsoluteTime elapsedTime, startTime = CFAbsoluteTimeGetCurrent();
-        
-        if (shouldRedrawUsingCoreGraphics) {
-            // For resized or incompatible image: redraw
-            imageData = UnsafeMutablePointer<GLubyte>.allocate(capacity:Int(widthToUseForTexture * heightToUseForTexture) * 4)
-
-            let genericRGBColorspace = CGColorSpaceCreateDeviceRGB()
+        try sharedImageProcessingContext.runOperationSynchronously {
+            //    CFAbsoluteTime elapsedTime, startTime = CFAbsoluteTimeGetCurrent();
 
-            let imageContext = CGContext(data:imageData, width:Int(widthToUseForTexture), height:Int(heightToUseForTexture), bitsPerComponent:8, bytesPerRow:Int(widthToUseForTexture) * 4, space:genericRGBColorspace,  bitmapInfo:CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)
-            //        CGContextSetBlendMode(imageContext, kCGBlendModeCopy); // From Technical Q&A QA1708: http://developer.apple.com/library/ios/#qa/qa1708/_index.html
-            imageContext?.draw(image, in:CGRect(x:0.0, y:0.0, width:CGFloat(widthToUseForTexture), height:CGFloat(heightToUseForTexture)))
-        } else {
-            // Access the raw image bytes directly
-            dataFromImageDataProvider = image.dataProvider?.data
-#if os(iOS)
-            imageData = UnsafeMutablePointer<GLubyte>(mutating:CFDataGetBytePtr(dataFromImageDataProvider))
-#else
-            imageData = UnsafeMutablePointer<GLubyte>(mutating:CFDataGetBytePtr(dataFromImageDataProvider)!)
-#endif
-        }
-        
-        sharedImageProcessingContext.runOperationSynchronously{
-            do {
-                // TODO: Alter orientation based on metadata from photo
-                self.imageFramebuffer = try Framebuffer(context:sharedImageProcessingContext, orientation:orientation, size:GLSize(width:widthToUseForTexture, height:heightToUseForTexture), textureOnly:true)
-            } catch {
-                fatalError("ERROR: Unable to initialize framebuffer of size (\(widthToUseForTexture), \(heightToUseForTexture)) with error: \(error)")
+            if (shouldRedrawUsingCoreGraphics) {
+                // For resized or incompatible image: redraw
+                imageData = UnsafeMutablePointer<GLubyte>.allocate(capacity:Int(widthToUseForTexture * heightToUseForTexture) * 4)
+
+                let genericRGBColorspace = CGColorSpaceCreateDeviceRGB()
+
+                let imageContext = CGContext(data: imageData, width: Int(widthToUseForTexture), height: Int(heightToUseForTexture), bitsPerComponent: 8, bytesPerRow: Int(widthToUseForTexture) * 4, space: genericRGBColorspace,  bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)
+                //        CGContextSetBlendMode(imageContext, kCGBlendModeCopy); // From Technical Q&A QA1708: http://developer.apple.com/library/ios/#qa/qa1708/_index.html
+                imageContext?.draw(image, in:CGRect(x:0.0, y:0.0, width:CGFloat(widthToUseForTexture), height:CGFloat(heightToUseForTexture)))
+            } else {
+                // Access the raw image bytes directly
+                guard let data = image.dataProvider?.data else { throw PictureInputError.dataProviderNilError }
+                dataFromImageDataProvider = data
+                imageData = UnsafeMutablePointer<GLubyte>(mutating:CFDataGetBytePtr(dataFromImageDataProvider))
             }
-            
-            glBindTexture(GLenum(GL_TEXTURE_2D), self.imageFramebuffer.texture)
+
+            // TODO: Alter orientation based on metadata from photo
+            self.imageFramebuffer = try Framebuffer(context:sharedImageProcessingContext, orientation:orientation, size:GLSize(width:widthToUseForTexture, height:heightToUseForTexture), textureOnly:true)
+            self.imageFramebuffer!.lock()
+
+            glBindTexture(GLenum(GL_TEXTURE_2D), self.imageFramebuffer!.texture)
             if (smoothlyScaleOutput) {
                 glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_MIN_FILTER), GL_LINEAR_MIPMAP_LINEAR)
             }
@@ -130,45 +146,53 @@ public class PictureInput: ImageSource {
     }
 
 #if canImport(UIKit)
-    public convenience init(image:UIImage, smoothlyScaleOutput:Bool = false, orientation:ImageOrientation = .portrait) {
-        self.init(image:image.cgImage!, smoothlyScaleOutput:smoothlyScaleOutput, orientation:orientation)
+    public convenience init(image:UIImage, smoothlyScaleOutput:Bool = false, orientation:ImageOrientation = .portrait) throws {
+        try self.init(image:image.cgImage!, smoothlyScaleOutput:smoothlyScaleOutput, orientation:orientation)
     }
 #else
     public convenience init(image:NSImage, smoothlyScaleOutput:Bool = false, orientation:ImageOrientation = .portrait) {
-        self.init(image:image.cgImage(forProposedRect:nil, context:nil, hints:nil)!, smoothlyScaleOutput:smoothlyScaleOutput, orientation:orientation)
+        try self.init(image:image.cgImage(forProposedRect:nil, context:nil, hints:nil)!, smoothlyScaleOutput:smoothlyScaleOutput, orientation:orientation)
     }
 #endif
 
-    public convenience init(imageName:String, smoothlyScaleOutput:Bool = false, orientation:ImageOrientation = .portrait) {
+    public convenience init(imageName:String, smoothlyScaleOutput:Bool = false, orientation:ImageOrientation = .portrait) throws {
 #if canImport(UIKit)
-        guard let image = UIImage(named:imageName) else { fatalError("No such image named: \(imageName) in your application bundle") }
-        self.init(image:image.cgImage!, smoothlyScaleOutput:smoothlyScaleOutput, orientation:orientation)
+        guard let image = UIImage(named:imageName) else { throw PictureInputError.noSuchImageError(imageName: imageName) }
+        try self.init(image:image.cgImage!, smoothlyScaleOutput:smoothlyScaleOutput, orientation:orientation)
 #else
-        guard let image = NSImage(named:NSImage.Name(imageName)) else { fatalError("No such image named: \(imageName) in your application bundle") }
-        self.init(image:image.cgImage(forProposedRect:nil, context:nil, hints:nil)!, smoothlyScaleOutput:smoothlyScaleOutput, orientation:orientation)
+        guard let image = NSImage(named:NSImage.Name(imageName)) else { throw PictureInputError.noSuchImageError(imageName: imageName) }
+        try self.init(image:image.cgImage(forProposedRect:nil, context:nil, hints:nil)!, smoothlyScaleOutput:smoothlyScaleOutput, orientation:orientation)
 #endif
     }
 
+    deinit {
+        self.imageFramebuffer?.unlock()
+    }
+
     public func processImage(synchronously:Bool = false) {
         if synchronously {
-            sharedImageProcessingContext.runOperationSynchronously{
-                sharedImageProcessingContext.makeCurrentContext()
-                self.updateTargetsWithFramebuffer(self.imageFramebuffer)
-                self.hasProcessedImage = true
+            sharedImageProcessingContext.runOperationSynchronously {
+                if let framebuffer = self.imageFramebuffer {
+                    sharedImageProcessingContext.makeCurrentContext()
+                    self.updateTargetsWithFramebuffer(framebuffer)
+                    self.hasProcessedImage = true
+                }
             }
         } else {
-            sharedImageProcessingContext.runOperationAsynchronously{
-                sharedImageProcessingContext.makeCurrentContext()
-                self.updateTargetsWithFramebuffer(self.imageFramebuffer)
-                self.hasProcessedImage = true
+            sharedImageProcessingContext.runOperationAsynchronously {
+                if let framebuffer = self.imageFramebuffer {
+                    sharedImageProcessingContext.makeCurrentContext()
+                    self.updateTargetsWithFramebuffer(framebuffer)
+                    self.hasProcessedImage = true
+                }
             }
         }
     }
 
     public func transmitPreviousImage(to target:ImageConsumer, atIndex:UInt) {
-        if hasProcessedImage {
-            imageFramebuffer.lock()
-            target.newFramebufferAvailable(imageFramebuffer, fromSourceIndex:atIndex)
+        if let framebuffer = self.imageFramebuffer, hasProcessedImage {
+            framebuffer.lock()
+            target.newFramebufferAvailable(framebuffer, fromSourceIndex:atIndex)
         }
     }
 }

framework/Source/Apple/PictureOutput.swift

@@ -64,7 +64,7 @@ public class PictureOutput: ImageConsumer {
         renderFramebuffer.unlock()
         guard let dataProvider = CGDataProvider(dataInfo:nil, data:data, size:imageByteSize, releaseData: dataProviderReleaseCallback) else {fatalError("Could not allocate a CGDataProvider")}
         let defaultRGBColorSpace = CGColorSpaceCreateDeviceRGB()
-        return CGImage(width:Int(framebuffer.size.width), height:Int(framebuffer.size.height), bitsPerComponent:8, bitsPerPixel:32, bytesPerRow:4 * Int(framebuffer.size.width), space:defaultRGBColorSpace, bitmapInfo:CGBitmapInfo() /*| CGImageAlphaInfo.Last*/, provider:dataProvider, decode:nil, shouldInterpolate:false, intent:.defaultIntent)!
+        return CGImage(width:Int(framebuffer.size.width), height:Int(framebuffer.size.height), bitsPerComponent:8, bitsPerPixel:32, bytesPerRow:4 * Int(framebuffer.size.width), space:defaultRGBColorSpace, bitmapInfo:CGBitmapInfo(rawValue: CGImageAlphaInfo.last.rawValue), provider:dataProvider, decode:nil, shouldInterpolate:false, intent:.defaultIntent)!
     }
 
     public func newFramebufferAvailable(_ framebuffer:Framebuffer, fromSourceIndex:UInt) {
@@ -140,15 +140,15 @@ public extension ImageSource {
 }
 
 public extension PlatformImageType {
-    func filterWithOperation<T:ImageProcessingOperation>(_ operation:T) -> PlatformImageType {
-        return filterWithPipeline{input, output in
+    func filterWithOperation<T:ImageProcessingOperation>(_ operation:T) throws -> PlatformImageType {
+        return try filterWithPipeline{input, output in
             input --> operation --> output
         }
     }
 
-    func filterWithPipeline(_ pipeline:(PictureInput, PictureOutput) -> ()) -> PlatformImageType {
+    func filterWithPipeline(_ pipeline:(PictureInput, PictureOutput) -> ()) throws -> PlatformImageType {
         var outputImage:PlatformImageType?
-        let picture = PictureInput(image:self)
+        let picture = try PictureInput(image:self)
         let pictureOutput = PictureOutput()
         pictureOutput.onlyCaptureNextFrame = true
         pictureOutput.imageAvailableCallback = {image in

framework/Source/Apple/RenderView-UIKit.swift

@@ -20,7 +20,7 @@ public class RenderView:UIView, ImageConsumer {
         return sharedImageProcessingContext.passthroughShader
     }()
 
-    // TODO: Need to set viewport to appropriate size, resize viewport on view reshape
+    private var internalLayer: CAEAGLLayer!
 
     required public init?(coder:NSCoder) {
         super.init(coder:coder)
@@ -37,20 +37,36 @@ public class RenderView:UIView, ImageConsumer {
             return CAEAGLLayer.self
         }
     }
+
+    override public var bounds: CGRect {
+        didSet {
+            // Check if the size changed
+            if(oldValue.size != self.bounds.size) {
+                // Destroy the displayFramebuffer so we render at the correct size for the next frame
+                sharedImageProcessingContext.runOperationAsynchronously{
+                    self.destroyDisplayFramebuffer()
+                }
+            }
+        }
+    }
 
     func commonInit() {
         self.contentScaleFactor = UIScreen.main.scale
 
         let eaglLayer = self.layer as! CAEAGLLayer
         eaglLayer.isOpaque = true
-        eaglLayer.drawableProperties = [String(describing: NSNumber(value:false)): kEAGLDrawablePropertyRetainedBacking, kEAGLColorFormatRGBA8: kEAGLDrawablePropertyColorFormat]
+        eaglLayer.drawableProperties = [kEAGLDrawablePropertyRetainedBacking: false, kEAGLDrawablePropertyColorFormat: kEAGLColorFormatRGBA8]
+
+        self.internalLayer = eaglLayer
     }
 
     deinit {
-        destroyDisplayFramebuffer()
+        sharedImageProcessingContext.runOperationSynchronously{
+            destroyDisplayFramebuffer()
+        }
     }
 
-    func createDisplayFramebuffer() {
+    func createDisplayFramebuffer() -> Bool {
         var newDisplayFramebuffer:GLuint = 0
         glGenFramebuffers(1, &newDisplayFramebuffer)
         displayFramebuffer = newDisplayFramebuffer
@@ -61,39 +77,53 @@ public class RenderView:UIView, ImageConsumer {
         displayRenderbuffer = newDisplayRenderbuffer
         glBindRenderbuffer(GLenum(GL_RENDERBUFFER), displayRenderbuffer!)
 
-        sharedImageProcessingContext.context.renderbufferStorage(Int(GL_RENDERBUFFER), from:self.layer as! CAEAGLLayer)
+        // Without the flush you will occasionally get a warning from UIKit and when that happens the RenderView just stays black.
+        // "CoreAnimation: [EAGLContext renderbufferStorage:fromDrawable:] was called from a non-main thread in an implicit transaction!
+        // Note that this may be unsafe without an explicit CATransaction or a call to [CATransaction flush]."
+        // I tried a transaction and that doesn't work and this is probably why --> http://danielkbx.com/post/108060601989/catransaction-flush
+        // Using flush is important because it guarantees the view is layed out at the correct size before it is drawn to since this is being done on a background thread.
+        // Its possible the size of the view was changed right before we got here and would result in us drawing to the view at the old size
+        // and then the view size would change to the new size at the next layout pass and distort our already drawn image.
+        // Since we do not call this function often we do not need to worry about the performance impact of calling flush.
+        CATransaction.flush()
+        sharedImageProcessingContext.context.renderbufferStorage(Int(GL_RENDERBUFFER), from:self.internalLayer)
 
         var backingWidth:GLint = 0
         var backingHeight:GLint = 0
         glGetRenderbufferParameteriv(GLenum(GL_RENDERBUFFER), GLenum(GL_RENDERBUFFER_WIDTH), &backingWidth)
         glGetRenderbufferParameteriv(GLenum(GL_RENDERBUFFER), GLenum(GL_RENDERBUFFER_HEIGHT), &backingHeight)
         backingSize = GLSize(width:backingWidth, height:backingHeight)
 
-        guard ((backingWidth > 0) && (backingHeight > 0)) else {
-            fatalError("View had a zero size")
+        guard (backingWidth > 0 && backingHeight > 0) else {
+            print("WARNING: View had a zero size")
+
+            if(self.internalLayer.bounds.width > 0 && self.internalLayer.bounds.height > 0) {
+                print("WARNING: View size \(self.internalLayer.bounds) may be too large ")
+            }
+            return false
         }
 
         glFramebufferRenderbuffer(GLenum(GL_FRAMEBUFFER), GLenum(GL_COLOR_ATTACHMENT0), GLenum(GL_RENDERBUFFER), displayRenderbuffer!)
 
         let status = glCheckFramebufferStatus(GLenum(GL_FRAMEBUFFER))
         if (status != GLenum(GL_FRAMEBUFFER_COMPLETE)) {
-            fatalError("Display framebuffer creation failed with error: \(FramebufferCreationError(errorCode:status))")
+            print("WARNING: Display framebuffer creation failed with error: \(FramebufferCreationError(errorCode:status))")
+            return false
         }
+
+        return true
     }
 
     func destroyDisplayFramebuffer() {
-        sharedImageProcessingContext.runOperationSynchronously{
-            if let displayFramebuffer = self.displayFramebuffer {
-                var temporaryFramebuffer = displayFramebuffer
-                glDeleteFramebuffers(1, &temporaryFramebuffer)
-                self.displayFramebuffer = nil
-            }
-            
-            if let displayRenderbuffer = self.displayRenderbuffer {
-                var temporaryRenderbuffer = displayRenderbuffer
-                glDeleteRenderbuffers(1, &temporaryRenderbuffer)
-                self.displayRenderbuffer = nil
-            }
+        if let displayFramebuffer = self.displayFramebuffer {
+            var temporaryFramebuffer = displayFramebuffer
+            glDeleteFramebuffers(1, &temporaryFramebuffer)
+            self.displayFramebuffer = nil
+        }
+        if let displayRenderbuffer = self.displayRenderbuffer {
+            var temporaryRenderbuffer = displayRenderbuffer
+            glDeleteRenderbuffers(1, &temporaryRenderbuffer)
+            self.displayRenderbuffer = nil
         }
     }
 
@@ -103,8 +133,10 @@ public class RenderView:UIView, ImageConsumer {
     }
 
     public func newFramebufferAvailable(_ framebuffer:Framebuffer, fromSourceIndex:UInt) {
-        if (displayFramebuffer == nil) {
-            self.createDisplayFramebuffer()
+        if (self.displayFramebuffer == nil && !self.createDisplayFramebuffer()) {
+            // Bail if we couldn't successfully create the displayFramebuffer
+            framebuffer.unlock()
+            return
         }
         self.activateDisplayFramebuffer()