Flutter实战图片组件演进之外接纹理解析

前言

其实Flutter本身已具备加载图片的能力，Image组件就满足网络图片、本地图片、文件图片的加载。那为什么我们还需要实现其他图片加载方案呢？其实是因为Flutter图片组件功能上存在一些缺陷：

• 图片缓存没有持久化能力，无网环境下不支持显示图片。
• 文件图片与原生环境不共用，导致图片资源文件重复。

因此为了满足日常开发需要和优化点，可以做点什么让图片组件功能达到满意的效果。接下来从Flutter原生组件再到外接纹理慢慢了解图片组件功能演进的过程。

Flutter原生图片组件

Flutter原生图片支持多种加载形式：

• Image.network（网络图片）
• Image.file （本地图片）
• Image.asset （文件图片）
• Image.memory （byte图片）

图片加载流程简要（网络图片为例）

• 第一步：网络图片加载形式以NetworkImage，内部由network_image.NetworkImage构成。

Image.network(
    String src, {
    ......省略不必要代码
  }) : image = ResizeImage.resizeIfNeeded(cacheWidth, cacheHeight, NetworkImage(src, scale: scale, headers: headers)),
    ...... 省略不必要代码
       super(key: key);
......
const factory NetworkImage(String url, { double scale, Map<String, String> headers }) = network_image.NetworkImage;

• 第二步：NetWorkImage实质上继承于ImageProvider，其他加载形式也是如此。ImageProvider是处理图片基本抽象类，继承它的加载类主要实现load方法执行不同形式加载过程。

abstract class ImageProvider<T> {
  const ImageProvider();
  .......
  @protected
  ImageStreamCompleter load(T key, DecoderCallback decode);
  .......
}

• 第三步：例如网络形式获取图片数据过程通过网络，通过Dart层网络请求HttpClient请求图片获取最终数据Uint8List。

class NetworkImage extends image_provider.ImageProvider<image_provider.NetworkImage> implements image_provider.NetworkImage {
    .......
    @override
  ImageStreamCompleter load(image_provider.NetworkImage key, image_provider.DecoderCallback decode) {
    final StreamController<ImageChunkEvent> chunkEvents = StreamController<ImageChunkEvent>();

    return MultiFrameImageStreamCompleter(
      codec: _loadAsync(key as NetworkImage, chunkEvents, decode),
      chunkEvents: chunkEvents.stream,
      scale: key.scale,
      informationCollector: () {
        return <DiagnosticsNode>[
          DiagnosticsProperty<image_provider.ImageProvider>('Image provider', this),
          DiagnosticsProperty<image_provider.NetworkImage>('Image key', key),
        ];
      },
    );
  }
  
  
  Future<ui.Codec> _loadAsync(
    NetworkImage key,
    StreamController<ImageChunkEvent> chunkEvents,
    image_provider.DecoderCallback decode,
  ) async {
    try {
      final Uri resolved = Uri.base.resolve(key.url);
      final HttpClientRequest request = await _httpClient.getUrl(resolved);
      headers?.forEach((String name, String value) {
        request.headers.add(name, value);
      });
      final HttpClientResponse response = await request.close();
      if (response.statusCode != HttpStatus.ok) {
        PaintingBinding.instance.imageCache.evict(key);
        throw image_provider.NetworkImageLoadException(statusCode: response.statusCode, uri: resolved);
      }

      final Uint8List bytes = await consolidateHttpClientResponseBytes(
        response,
        onBytesReceived: (int cumulative, int total) {
          chunkEvents.add(ImageChunkEvent(
            cumulativeBytesLoaded: cumulative,
            expectedTotalBytes: total,
          ));
        },
      );
      if (bytes.lengthInBytes == 0)
        throw Exception('NetworkImage is an empty file: $resolved');
      return decode(bytes);
    } finally {
      chunkEvents.close();
    }
  }
}

• 第四步：获取到图片Uint8List数据之后就是解码过程。通过DecoderCallback回调方法得到图片原始数据之后交付给全局单例解码器PaintingBinding.instance.instantiateImageCodec，然后由引擎层C++的instantiateImageCodec处理数据返回可被Flutter层渲染展示Image数据。

 final ImageStreamCompleter completer = PaintingBinding.instance.imageCache.putIfAbsent(
      key,
      () => load(key, PaintingBinding.instance.instantiateImageCodec),
      onError: handleError,
    );
 Future<ui.Codec> instantiateImageCodec(Uint8List bytes, {
    int cacheWidth,
    int cacheHeight,
  }) {
    assert(cacheWidth == null || cacheWidth > 0);
    assert(cacheHeight == null || cacheHeight > 0);
    return ui.instantiateImageCodec(
      bytes,
      targetWidth: cacheWidth,
      targetHeight: cacheHeight,
    );
  }    
String _instantiateImageCodec(Uint8List list, _Callback<Codec> callback, _ImageInfo imageInfo, int targetWidth, int targetHeight)
  native 'instantiateImageCodec';

• 第五步：引擎层c++解码器具体在codec.cc中，调用了Skia的SkCodec对图片数据做处理。经过解码器内部处理后执行ToDart将ui_codec返回到Dart层。

/// Dart层代码
_String _instantiateImageCodec(Uint8List list, _Callback<Codec> callback, _ImageInfo imageInfo, int targetWidth, int targetHeight)
  native 'instantiateImageCodec';
/// c++层代码
static void InstantiateImageCodec(Dart_NativeArguments args) {
  UIDartState::ThrowIfUIOperationsProhibited();
  Dart_Handle callback_handle = Dart_GetNativeArgument(args, 1);
  .......省略部分代码
  Dart_Handle image_info_handle = Dart_GetNativeArgument(args, 2);

  std::optional<ImageDecoder::ImageInfo> image_info;
  /// 图片信息是否为空，不为空做一些处理
  if (!Dart_IsNull(image_info_handle)) {
    auto image_info_results = ConvertImageInfo(image_info_handle, args);
    if (auto value =
            std::get_if<ImageDecoder::ImageInfo>(&image_info_results)) {
      image_info = *value;
    } else if (auto error = std::get_if<std::string>(&image_info_results)) {
      Dart_SetReturnValue(args, tonic::ToDart(*error));
      return;
    }
  }

  sk_sp<SkData> buffer;

  {
    /// 处理图片数据
    Dart_Handle exception = nullptr;
    tonic::Uint8List list =
        tonic::DartConverter<tonic::Uint8List>::FromArguments(args, 0,
                                                              exception);
    if (exception) {
      Dart_SetReturnValue(args, exception);
      return;
    }
    /// 图片数据做拷贝
    buffer = MakeSkDataWithCopy(list.data(), list.num_elements());
  }

  if (image_info) {
    const auto expected_size =
        image_info->row_bytes * image_info->sk_info.height();
    if (buffer->size() < expected_size) {
      Dart_SetReturnValue(
          args, ToDart("Pixel buffer size does not match image size"));
      return;
    }
  }
   /// 获取图片目标宽高
  const int targetWidth =
      tonic::DartConverter<int>::FromDart(Dart_GetNativeArgument(args, 3));
  const int targetHeight =
      tonic::DartConverter<int>::FromDart(Dart_GetNativeArgument(args, 4));

  std::unique_ptr<SkCodec> codec;
  bool single_frame;
  if (image_info) {
    single_frame = true;
  } else {
    /// 底层解码器使用的是SkCodec解码器，Android底层同样使用的是它。
    codec = SkCodec::MakeFromData(buffer);
    if (!codec) {
      Dart_SetReturnValue(args, ToDart("Could not instantiate image codec."));
      return;
    }
    single_frame = codec->getFrameCount() == 1;
  }
   /// 解码器同时解码后得出帧数信息，判断图片是否为动图
  fml::RefPtr<Codec> ui_codec;

  if (single_frame) {
    ImageDecoder::ImageDescriptor descriptor;
    descriptor.decompressed_image_info = image_info;

    if (targetWidth > 0) {
      descriptor.target_width = targetWidth;
    }
    if (targetHeight > 0) {
      descriptor.target_height = targetHeight;
    }
    descriptor.data = std::move(buffer);

    ui_codec = fml::MakeRefCounted<SingleFrameCodec>(std::move(descriptor));
  } else {
    ui_codec = fml::MakeRefCounted<MultiFrameCodec>(std::move(codec));
  }
  /// 最后将解码器结果返回到Dart层
  tonic::DartInvoke(callback_handle, {ToDart(ui_codec)});
}

外接纹理渲染图片

Flutter中有一个叫做Texture组件,该组件只有唯一入参textureId，寥寥无几的几行代码就实现外接纹理着实让人摸不清头脑。在分析外接纹理原理之前先简单了解外接纹理渲染图片功能实现。

Texture组件使用

• Java层通过Channel插件PluginRegistry.Registrar创建Surface、textureId。

/// 插件接口获取texture注册器
TextureRegistry textureRegistry = registrar.textures();
/// 创建Texture实例
TextureRegistry.SurfaceTextureEntry surfaceTextureEntry = textureRegistry.createSurfaceTexture();
long textureId = surfaceTextureEntry.id();
SurfaceTexture surfaceTexture = surfaceTextureEntry.surfaceTexture();
/// 获取图片地址
String url = call.argument("url");
...... 省略图片请求加载过程
/// 创建Surface实例加载surfaceTexture
Surface surface = new Surface(surfaceTexture);
/// 画布绘制bitmap 纹理映射
Canvas canvas = surface.lockCanvas(rect);
canvas.drawBitmap(bitmap, null, rect, null);
bitmap.recycle();
surface.unlockCanvasAndPost(canvas);
/// Dart返回textureId
Map<String, Object> maps = new HashMap<>();
maps.put("textureId", textureId);
result.success(maps);

• Dart层创建MethodChannel，向Native层传递加载图片路径。

  static const MethodChannel _channel = const MethodChannel('texture_channel');
  /// 原始加载图片接口
  static Future<Map> loadTexture({String url}) async {
    var args = <String, dynamic>{
      "url": url,
    };
    return await _channel.invokeMethod("loadTexture", args);
  }
  /// 执行加载
  Map _textureResult = await TexturePlugin.loadTexture(
      url: _uri.toString(),
      width: url.width,
      height: url.height,
    );
   /// 返回Native生成的textureId
   int id = _textureResult['textureId'];
   /// 实例化texture组件 显示图片
   Texture( textureId: id);

代码解析

Dart层

从源码上可以看到Texture会创建渲染对象TextureBox。TextureBox会去绘制TextureLayer，TextureLayer则通过ui.SceneBuilder向Scene添加纹理，最终是调用引擎层SceneBuilder_addTexture方法实现纹理渲染。

• Texture

class Texture extends LeafRenderObjectWidget {
  const Texture({
    Key key,
    @required this.textureId,
  }) : assert(textureId != null),
       super(key: key);
       
  final int textureId;
  
  @override
  TextureBox createRenderObject(BuildContext context) => TextureBox(textureId: textureId);

  @override
  void updateRenderObject(BuildContext context, TextureBox renderObject) {
    renderObject.textureId = textureId;
  }
}

• TextureBox

class TextureBox extends RenderBox {
  TextureBox({ @required int textureId })
    : assert(textureId != null),
      _textureId = textureId;
  ...... 省略代码
  @override
  void paint(PaintingContext context, Offset offset) {
    if (_textureId == null)
      return;
    context.addLayer(TextureLayer(
      rect: Rect.fromLTWH(offset.dx, offset.dy, size.width, size.height),
      textureId: _textureId,
    ));
  }
}

• TextureLayer

class TextureLayer extends Layer {
  TextureLayer({
    @required this.rect,
    @required this.textureId,
    this.freeze = false,
  }) : assert(rect != null),
       assert(textureId != null);

  ......
  @override
  void addToScene(ui.SceneBuilder builder, [ Offset layerOffset = Offset.zero ]) {
    final Rect shiftedRect = layerOffset == Offset.zero ? rect : rect.shift(layerOffset);
    builder.addTexture(
      textureId,
      offset: shiftedRect.topLeft,
      width: shiftedRect.width,
      height: shiftedRect.height,
      freeze: freeze,
    );
  }
}

• SceneBuilder

class SceneBuilder extends NativeFieldWrapperClass2 {
 
 void addTexture( int textureId, { Offset offset = Offset.zero, double width = 0.0, double height = 0.0, bool freeze = false, }) {
   _addTexture(offset.dx, offset.dy, width, height, textureId, freeze);
 }
   /// SceneBuilder_addTexture对应scene_builder.cc下的SceneBuilder::addTexture方法
 void _addTexture(double dx, double dy, double width, double height, int textureId, bool freeze)
     native 'SceneBuilder_addTexture';
    

• scene_builder.cc

void SceneBuilder::addTexture(double dx, double dy, double width, double height, int64_t textureId, bool freeze) {
  auto layer = std::make_unique<flutter::TextureLayer>(
      SkPoint::Make(dx, dy), SkSize::Make(width, height), textureId, freeze);
  AddLayer(std::move(layer));
}

• texture_layer.cc

/// 创建纹理层对象
TextureLayer::TextureLayer(const SkPoint& offset,
                           const SkSize& size,
                           int64_t texture_id,
                           bool freeze)
    : offset_(offset), size_(size), texture_id_(texture_id), freeze_(freeze) {}
/// 纹理对象绘制方法
void TextureLayer::Paint(PaintContext& context) const {
  TRACE_EVENT0("flutter", "TextureLayer::Paint");
  /// texture对象绘制时会从GetTexture方法的map中找到纹理对象进行绘制。
  std::shared_ptr<Texture> texture =
      context.texture_registry.GetTexture(texture_id_);
  if (!texture) {
    TRACE_EVENT_INSTANT0("flutter", "null texture");
    return;
  }
  texture->Paint(*context.leaf_nodes_canvas, paint_bounds(), freeze_,
                 context.gr_context);
}

Java层

• FlutterRenderer

public class FlutterRenderer implements TextureRegistry {

  public FlutterRenderer(@NonNull FlutterJNI flutterJNI) {
    this.flutterJNI = flutterJNI;
    this.flutterJNI.addIsDisplayingFlutterUiListener(flutterUiDisplayListener);
  }
    @Override
  public SurfaceTextureEntry createSurfaceTexture() {
   /// 创建SurfaceTexture
    final SurfaceTexture surfaceTexture = new SurfaceTexture(0);
    surfaceTexture.detachFromGLContext();
    final SurfaceTextureRegistryEntry entry =
        new SurfaceTextureRegistryEntry(nextTextureId.getAndIncrement(), surfaceTexture);
    
    registerTexture(entry.id(), surfaceTexture);
    return entry;
  }
  ///向Native JNI层注册
  private void registerTexture(long textureId, @NonNull SurfaceTexture surfaceTexture) {
    flutterJNI.registerTexture(textureId, surfaceTexture);
  }
}

• FlutterJNI

  public void registerTexture(long textureId, @NonNull SurfaceTexture surfaceTexture) {
    /// 这里需要注意必须在主线程执行否则会报错
    ensureRunningOnMainThread();
    ensureAttachedToNative();
    nativeRegisterTexture(nativePlatformViewId, textureId, surfaceTexture);
  }

C++层

• platform_view_android_jni.cc

{
          .name = "nativeRegisterTexture",
          .signature = "(JJLandroid/graphics/SurfaceTexture;)V",
          .fnPtr = reinterpret_cast<void*>(&RegisterTexture),
}

static void RegisterTexture(JNIEnv* env,
                            jobject jcaller,
                            jlong shell_holder,
                            jlong texture_id,
                            jobject surface_texture) {
  ANDROID_SHELL_HOLDER->GetPlatformView()->RegisterExternalTexture(
      static_cast<int64_t>(texture_id),                        //
      fml::jni::JavaObjectWeakGlobalRef(env, surface_texture)  //
  );
}

• platform_view_android.cc

void PlatformViewAndroid::RegisterExternalTexture(
    int64_t texture_id,
    const fml::jni::JavaObjectWeakGlobalRef& surface_texture) {
  RegisterTexture(
      std::make_shared<AndroidExternalTextureGL>(texture_id, surface_texture));
}

• texture.cc

// 注册纹理方法
void TextureRegistry::RegisterTexture(std::shared_ptr<Texture> texture) {
  if (!texture) {
    return;
  }
  // 内部map保存texture实例
  mapping_[texture->Id()] = texture;
}
// 获取到纹理对象，从map中提取
std::shared_ptr<Texture> TextureRegistry::GetTexture(int64_t id) {
  auto it = mapping_.find(id);
  return it != mapping_.end() ? it->second : nullptr;
}

• android_external_texture_gl.cc

/// 外接纹理对象实例
AndroidExternalTextureGL::AndroidExternalTextureGL(
    int64_t id,
    const fml::jni::JavaObjectWeakGlobalRef& surfaceTexture)
    : Texture(id), surface_texture_(surfaceTexture), transform(SkMatrix::I()) {}

AndroidExternalTextureGL::~AndroidExternalTextureGL() {
  if (state_ == AttachmentState::attached) {
    glDeleteTextures(1, &texture_name_);
  }
}
/// 绘制方法
void AndroidExternalTextureGL::Paint(SkCanvas& canvas,
                                     const SkRect& bounds,
                                     bool freeze,
                                     GrContext* context) {
  if (state_ == AttachmentState::detached) {
    return;
  }
  if (state_ == AttachmentState::uninitialized) {
    glGenTextures(1, &texture_name_);
    Attach(static_cast<jint>(texture_name_));
    state_ = AttachmentState::attached;
  }
  if (!freeze && new_frame_ready_) {
    Update();
    new_frame_ready_ = false;
  }
  GrGLTextureInfo textureInfo = {GL_TEXTURE_EXTERNAL_OES, texture_name_,
                                 GL_RGBA8_OES};
  GrBackendTexture backendTexture(1, 1, GrMipMapped::kNo, textureInfo);
  sk_sp<SkImage> image = SkImage::MakeFromTexture(
      canvas.getGrContext(), backendTexture, kTopLeft_GrSurfaceOrigin,
      kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr);
  if (image) {
    SkAutoCanvasRestore autoRestore(&canvas, true);
    canvas.translate(bounds.x(), bounds.y());
    canvas.scale(bounds.width(), bounds.height());
    if (!transform.isIdentity()) {
      SkMatrix transformAroundCenter(transform);

      transformAroundCenter.preTranslate(-0.5, -0.5);
      transformAroundCenter.postScale(1, -1);
      transformAroundCenter.postTranslate(0.5, 0.5);
      canvas.concat(transformAroundCenter);
    }
    canvas.drawImage(image, 0, 0);
  }
}

• ①Java层FlutterRenderer创建SurfaceTexture和textureId。
• ②将urfaceTexture和textureId通过JNI向引擎层注册
• ③向引擎注册过程中通过层层方法最后在texture.cc的TextureRegistry由map以键值对形式缓存实例对象。
• ④将需要显示图片在SurfaceTexture上离屏渲染。
• ⑤Java层创建的textureId通过Channel传递到Dart层作为Texture组件入参。
• ⑥Dart的Texture组件接收textureId入参后向下层组件实例化。
• ⑦在SceneBuilder调用addTexture时执行引擎层创建TextureLayer。
• ⑧最终在texture.cc中TextureRegistry的map根据TextureId获取SurfaceTexture实例。

Image VS Texture

Image组件和Texture组件实质上都是对RenderBox的实现，已知Flutter渲染树实际上就是RenderObject合并为Layer最终通过Flutter引擎进行绘制上屏显示页面内容。两者区别在于实现RenderObject的paint有所不同，Image的实现将ui.Image内容绘制在ui.Canvas上，Texture是将TextureLayer添加到ui.Scene中。不同之处就在于Image做了绘制操作，Texture是做添加操作，纹理方案的图片加载和绘制完全由原生平台完成。

🚀纹理完整代码看这里🚀

参考

• 万万没想到-Flutter这样外接纹理
• 闲鱼Flutter图片框架架构演进