本文使用JNI技术在Android平台部署深度学习模型，并使用MNN框架进行模型推理。

模型及C++程序准备

mnist-mnn

Android环境配置

1. 打开Android studio， 创建一个Native C++工程，并配置OpenCV。
   在Android中使用OpenCV

2. 在PC上编译MNN-Android的动态链接库
   MNN安装和编译

3. CMakeLists.txt编写
   在jni中编译C/C++程序有两种方法：一是使用ndk-build(需要配置.mk文件)，二是使用CMake，本文使用CMake编译的方法。

cmake_minimum_required(VERSION 3.4.1)

# Creates and names a library, sets it as either STATIC
# or SHARED, and provides the relative paths to its source code.
# You can define multiple libraries, and CMake builds them for you.
# Gradle automatically packages shared libraries with your APK.
# opencv
set( OpenCV_DIR /home/yinliang/software/OpenCV-android-sdk/sdk/native/jni )
find_package(OpenCV REQUIRED)
# MNN_DIR为自己安装的MNN的路径
set(MNN_DIR /home/yinliang/software/MNN)
# mnn的头文件
include_directories(${MNN_DIR}/include)
include_directories(${MNN_DIR}/include/MNN)
include_directories(${MNN_DIR}/tools)
include_directories(${MNN_DIR}/tools/cpp)
include_directories(${MNN_DIR}/source)
include_directories(${MNN_DIR}/source/backend)
include_directories(${MNN_DIR}/source/core)
# 这个是自己定义的.h文件
include_directories(get_result.h)
# 链接mnn的动态库，这里编译的是64位的，对应Android里面的arm64-v8a架构
aux_source_directory(. SRCS)
add_library( # Sets the name of the library.
        native-lib
        # Sets the library as a shared library.
        SHARED
        # Provides a relative path to your source file(s).
        ${SRCS})

find_library( # Sets the name of the path variable.
        log-lib
        log)
# 需要把libMNN.so放到工程文件里来，具体位置在 app/libs下，放在工程外好像不行
set(dis_DIR ../../../../libs)
add_library(
        MNN
        SHARED
        IMPORTED
)
set_target_properties(
        MNN
        PROPERTIES IMPORTED_LOCATION
        ${dis_DIR}/arm64-v8a/libMNN.so
)
# 代码主要依赖opencv和mnn两个库，这里链接一下
target_link_libraries( # Specifies the target library.
        native-lib
        # Links the target library to the log library
        # included in the NDK.
        ${log-lib}
        MNN
        jnigraphics
        ${OpenCV_LIBS})

5. 修改app下的build.gradle文件
   添加以下内容，不然无法成功链接到libMNN.so

sourceSets {
    
            main{
    
                jniLibs.srcDirs=['libs']
            }
        }

完整的build.gradle为：

apply plugin: 'com.android.application'

android {
    
    compileSdkVersion 30
    defaultConfig {
    
        applicationId "com.mnn.mnist"
        minSdkVersion 25
        targetSdkVersion 26
        versionCode 1
        versionName "1.0"
        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
        sourceSets {
    
            main{
    
                jniLibs.srcDirs=['libs']
            }
        }
        externalNativeBuild {
    
            cmake {
    
                cppFlags "-std=c++14"
                arguments "-DANDROID_STL=c++_shared"
                abiFilters  "arm64-v8a"
            }
        }
    }
    buildTypes {
    
        release {
    
            minifyEnabled false
            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
        }
    }

    externalNativeBuild {
    
        cmake {
    
            path "src/main/cpp/CMakeLists.txt"
            version "3.10.2"

        }
    }
}
dependencies {
    
    implementation fileTree(dir: 'libs', include: ['*.jar'])
    implementation 'androidx.appcompat:appcompat:1.0.2'
    implementation 'androidx.constraintlayout:constraintlayout:1.1.3'
    testImplementation 'junit:junit:4.12'
    androidTestImplementation 'androidx.test:runner:1.1.1'
    androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1'
}

编写native-lib.cpp

1. 在src/main/cpp下新建一个get_result.cpp文件，实现MNN的前向推理过程。

//
// Created by yinliang on 20-8-17.
//
#include <jni.h>
#include <string>
#include <iostream>
#include <stdio.h>
#include <math.h>

#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgcodecs/imgcodecs.hpp>

#include "Backend.hpp"
#include "Interpreter.hpp"
#include "MNNDefine.h"
#include "Interpreter.hpp"
#include "Tensor.hpp"


using namespace MNN;
using namespace std;
using namespace cv;

int mnist(Mat image_src, const char* model_name){
    

    // const char* model_name = "/home/yinliang/works/C/MNN-APPLICATIONS/applications/mnist/onnx/jni/graphs/mnist.mnn";
    int forward = MNN_FORWARD_CPU;
    // int forward = MNN_FORWARD_OPENCL;

    int precision  = 2;
    int power      = 0;
    int memory     = 0;
    int threads    = 1;
    int INPUT_SIZE = 28;

    cv::Mat raw_image = image_src;
    cv::Mat image;
    cv::resize(raw_image, image, cv::Size(INPUT_SIZE, INPUT_SIZE));
    // cout<<"model_path:" << model_name<<endl;
    // 1. 创建Interpreter, 通过磁盘文件创建: static Interpreter* createFromFile(const char* file);
    std::shared_ptr<Interpreter> net(Interpreter::createFromFile(model_name));
    MNN::ScheduleConfig config;
    // 2. 调度配置,
    // numThread决定并发数的多少，但具体线程数和并发效率，不完全取决于numThread
    // 推理时，主选后端由type指定，默认为CPU。在主选后端不支持模型中的算子时，启用由backupType指定的备选后端。
    config.numThread = threads;
    config.type      = static_cast<MNNForwardType>(forward);
    MNN::BackendConfig backendConfig;
    // 3. 后端配置
    // memory、power、precision分别为内存、功耗和精度偏好
    backendConfig.precision = (MNN::BackendConfig::PrecisionMode)precision;
    backendConfig.power = (MNN::BackendConfig::PowerMode) power;
    backendConfig.memory = (MNN::BackendConfig::MemoryMode) memory;
    config.backendConfig = &backendConfig;
    // 4. 创建session
    auto session = net->createSession(config);
    net->releaseModel();

    clock_t start = clock();
    // preprocessing
    image.convertTo(image, CV_32FC3);
    image = image / 255.0f;
    // 5. 输入数据
    // wrapping input tensor, convert nhwc to nchw
    std::vector<int> dims{
    1, INPUT_SIZE, INPUT_SIZE, 3};
    auto nhwc_Tensor = MNN::Tensor::create<float>(dims, NULL, MNN::Tensor::TENSORFLOW);
    auto nhwc_data   = nhwc_Tensor->host<float>();
    auto nhwc_size   = nhwc_Tensor->size();
    ::memcpy(nhwc_data, image.data, nhwc_size);

    std::string input_tensor = "data";
    // 获取输入tensor
    // 拷贝数据, 通过这类拷贝数据的方式，用户只需要关注自己创建的tensor的数据布局，
    // copyFromHostTensor会负责处理数据布局上的转换（如需）和后端间的数据拷贝（如需）。
    auto inputTensor  = net->getSessionInput(session, nullptr);
    inputTensor->copyFromHostTensor(nhwc_Tensor);

    // 6. 运行会话
    net->runSession(session);

    // 7. 获取输出
    std::string output_tensor_name0 = "dense1_fwd";
    // 获取输出tensor
    MNN::Tensor *tensor_scores  = net->getSessionOutput(session, output_tensor_name0.c_str());

    MNN::Tensor tensor_scores_host(tensor_scores, tensor_scores->getDimensionType());
    // 拷贝数据
    tensor_scores->copyToHostTensor(&tensor_scores_host);

    // post processing steps
    auto scores_dataPtr  = tensor_scores_host.host<float>();

    // softmax
    float exp_sum = 0.0f;
    for (int i = 0; i < 10; ++i)
    {
    
        float val = scores_dataPtr[i];
        exp_sum += val;
    }
    // get result idx
    int  idx = 0;
    float max_prob = -10.0f;
    for (int i = 0; i < 10; ++i)
    {
    
        float val  = scores_dataPtr[i];
        float prob = val / exp_sum;
        if (prob > max_prob)
        {
    
            max_prob = prob;
            idx      = i;
        }
    }

    // printf("the result is %d\n", idx);
    return idx;
}

函数的输入为一个Mat类型的图像，const char*类型的模型地址，输出为识别结果。

2. 编写对应的.h文件
   在相同目录下创建get_result.cpp对应的头文件。

#include <jni.h>
#include <string>
#include <iostream>
#include <stdio.h>
#include <math.h>

#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgcodecs/imgcodecs.hpp>

#include "Backend.hpp"
#include "Interpreter.hpp"
#include "MNNDefine.h"
#include "Interpreter.hpp"
#include "Tensor.hpp"

using namespace MNN;
using namespace std;
using namespace cv;

int mnist(Mat image_src, const char* model_name);

3. 编写native-lib.cpp
   定义jni接口函数，也就是我们最后在Android端可以调用的本地方法，函数的参数类型都是jni特有的类型，可参考jni技术简介

#include <jni.h>
#include <string>
#include <android/bitmap.h>
#include <opencv2/opencv.hpp>
#include "get_result.h"
#include "stdio.h"
#include "stdlib.h"

extern "C" JNIEXPORT jstring JNICALL
Java_com_mnn_mnist_MainActivity_mnistJNI (JNIEnv *env, jobject obj, jobject bitmap, jstring jstr){
    

    AndroidBitmapInfo info;
    void *pixels;
    CV_Assert(AndroidBitmap_getInfo(env, bitmap, &info) >= 0);
    CV_Assert(info.format == ANDROID_BITMAP_FORMAT_RGBA_8888 ||
              info.format == ANDROID_BITMAP_FORMAT_RGB_565);
    CV_Assert(AndroidBitmap_lockPixels(env, bitmap, &pixels) >= 0);
    CV_Assert(pixels);
    if (info.format == ANDROID_BITMAP_FORMAT_RGBA_8888) {
    
        Mat temp(info.height, info.width, CV_8UC4, pixels);
        Mat temp2 = temp.clone();
		//将jstring类型转换成C++里的const char*类型
        const char *path = env->GetStringUTFChars(jstr, 0);

        Mat RGB;
        //先将图像格式由BGRA转换成RGB，不然识别结果不对
        cvtColor(temp2, RGB, COLOR_RGBA2RGB);
        //调用之前定义好的mnist()方法，识别文字图像
        int result = mnist(RGB, path);
        //将图像转回RGBA格式，Android端才可以显示
        Mat show(info.height, info.width, CV_8UC4, pixels);
        cvtColor(RGB, temp, COLOR_RGB2RGBA);
        //将int类型的识别结果转成jstring类型，并返回
        string re_reco = to_string(result);
        const char* ss = re_reco.c_str();
        char cap[12];
        strcpy(cap, ss);
        return (env)->NewStringUTF(cap);;

    } else {
    
        Mat temp(info.height, info.width, CV_8UC2, pixels);
        
    }
    AndroidBitmap_unlockPixels(env, bitmap);
}

Android端调用

由于不会Android开发，这部分代码很粗糙，能正确运行，但是不够优雅。

package com.mnn.mnist;

import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;

import android.Manifest;
import android.content.res.AssetManager;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.os.Bundle;
import android.os.Environment;
import android.view.View;
import android.widget.ImageView;
import android.widget.TextView;
import android.widget.Toast;

import java.io.File;

import static android.content.pm.PackageManager.PERMISSION_GRANTED;


public class MainActivity extends AppCompatActivity implements View.OnClickListener {
    
    //定义两个控件，分别用来显示图像和文本
    private ImageView imageView;
    private TextView textView;
    // 加载生成的动态链接库
    // Used to load the 'native-lib' library on application startup.
    static {
    
        System.loadLibrary("native-lib");
    }
    // 声明JNI函数，对应native-lib.cpp里定义的函数
    native String mnistJNI(Object bitmap, String str);

    @Override
    protected void onCreate(Bundle savedInstanceState) {
    
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);
        imageView = findViewById(R.id.imageView);
        findViewById(R.id.show).setOnClickListener((View.OnClickListener) this);
        findViewById(R.id.process).setOnClickListener((View.OnClickListener) this);
        findViewById(R.id.gray).setOnClickListener((View.OnClickListener) this);

        textView = findViewById(R.id.textView);
        findViewById(R.id.textView).setOnClickListener((View.OnClickListener) this);

    }

    @Override
    public void onClick(View v) {
    
    // show为一个button，只用来显示一下图像
        if (v.getId() == R.id.show) {
    
        //放一张图像到res/drawable目录下，并命名为test.jpg
        //读取图像，在Android里对应的类型为Bitmap
            Bitmap bitmap = BitmapFactory.decodeResource(getResources(), R.drawable.test);
            //显示图像
            imageView.setImageBitmap(bitmap);
        } else {
    
         // 
            Bitmap bitmap = BitmapFactory.decodeResource(getResources(), R.drawable.test);
            //把模型从Assets拷贝到Cache，再获取模型的路径，供C++读取
            copyAssetAndWrite("mnist.mnn");
            File dataFile = new File(getCacheDir(),"mnist.mnn");
            //System.out.println("模型路径:" + dataFile.getAbsolutePath());
            imageView.setImageBitmap(bitmap);
            textView.setText(mnistJNI(bitmap, dataFile.getAbsolutePath()));
        }
    }

    @Override
    public void onPointerCaptureChanged(boolean hasCapture) {
    
    }
    private boolean copyAssetAndWrite(String fileName){
    
        try {
    
            File cacheDir=getCacheDir();
            if (!cacheDir.exists()){
    
                cacheDir.mkdirs();
            }
            File outFile =new File(cacheDir,fileName);
            if (!outFile.exists()){
    
                boolean res=outFile.createNewFile();
                if (!res){
    
                    return false;
                }
            }else {
    
                if (outFile.length()>10){
    //表示已经写入一次
                    return true;
                }
            }
            InputStream is=getAssets().open(fileName);
            FileOutputStream fos = new FileOutputStream(outFile);
            byte[] buffer = new byte[1024];
            int byteCount;
            while ((byteCount = is.read(buffer)) != -1) {
    
                fos.write(buffer, 0, byteCount);
            }
            fos.flush();
            is.close();
            fos.close();
            return true;
        } catch (IOException e) {
    
            e.printStackTrace();
        }

        return false;
    }
}

对应的界面布局文件

<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:layout_width="match_parent"
    android:layout_height="match_parent">

    <ImageView
        android:id="@+id/imageView"
        android:layout_width="match_parent"
        android:layout_height="match_parent" />

    <LinearLayout
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        android:layout_alignParentBottom="true"
        android:orientation="horizontal">

        <Button
            android:id="@+id/show"
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            android:layout_weight="1"
            android:text="show" />

        <Button
            android:id="@+id/process"
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            android:layout_weight="1"
            android:text="mnist" />

        <Button
            android:id="@+id/gray"
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            android:layout_weight="1"
            android:text="gray" />
    </LinearLayout>

<TextView
    android:id="@+id/textView"
    android:layout_width="match_parent"
    android:layout_height="wrap_content"
    android:gravity="center"
    android:textSize="24sp"
    android:textColor="#00ff00"
    android:text="result"
    />

</RelativeLayout>

TODO

1. 输入图像为res里面存放的固定图像，不知道怎么从相册里选取一张图识别或是拍照识别。