Elasticsearch7.8.0集成IK分词器改源码实现MySql5.7.2实现动态词库实时更新

在github上的IK源码描述文件中对远程扩展词库是这样描述的

permission java.net.SocketPermission "*", "connect,resolve";
复制

<dependency>
    <groupId>mysql</groupId>
    <artifactId>mysql-connector-java</artifactId>
    <version>5.1.38</version>
</dependency>
复制

public class HotDictReloadThread {
    private static final Logger log = ESPluginLoggerFactory.getLogger(HotDictReloadThread.class.getName());
    public void initial(){
        while (true) {
            log.info("正在调用HotDictReloadThread...");
            Dictionary.getSingleton().reLoadMainDict();
        }
    }
}
复制

pool.execute(() -> new HotDictReloadThread().initial());
复制

CREATE TABLE `es_lexicon`  (
`lexicon_id` bigint(8) NOT NULL AUTO_INCREMENT COMMENT '词库id',
`lexicon_text` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '词条关键词',
`lexicon_type` int(1) NOT NULL DEFAULT 0 COMMENT '0扩展词库 1停用词库',
`lexicon_status` int(1) NOT NULL DEFAULT 0 COMMENT '词条状态 0正常 1暂停使用',
`del_flag` int(1) NOT NULL DEFAULT 0 COMMENT '作废标志 0正常 1作废',
`create_time` datetime(0) NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
PRIMARY KEY (`lexicon_id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = 'ES远程扩展词库表' ROW_FORMAT = Dynamic;
复制

# 数据库地址
jdbc.url=jdbc:mysql://127.0.0.1:3306/test?serverTimezone=GMT&autoReconnect=true&useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useAffectedRows=true&useSSL=false
# 数据库用户名
jdbc.user=root
# 数据库密码
jdbc.password=123456
# 数据库查询扩展词库sql语句
jdbc.reload.sql=select gel.lexicon_text as word from es_lexicon gel where gel.lexicon_type = 0 and gel.lexicon_status = 0 and gel.del_flag = 0 order by gel.lexicon_id desc
# 数据库查询停用词sql语句
jdbc.reload.stopword.sql=select gel.lexicon_text as word from ges_lexicon gel where gel.lexicon_type = 1 and gel.lexicon_status = 0 and gel.del_flag = 0 order by gel.lexicon_id desc
# 数据库查询间隔时间 每隔10秒请求一次
jdbc.reload.interval=10
复制

/**
* 从MySql中加载动态词库
*/
private void loadMySQLExtDict() {
    Connection conn = null;
    Statement stmt = null;
    ResultSet rs = null;
    try {
        Path file = PathUtils.get(getDictRoot(), "jdbc-reload.properties");
        props.load(new FileInputStream(file.toFile()));
        logger.info("[==========]jdbc-reload.properties");
        for(Object key : props.keySet()) {
            logger.info("[==========]" + key + "=" + props.getProperty(String.valueOf(key)));
        }
    
        logger.info("[==========]query hot dict from mysql, " + props.getProperty("jdbc.reload.sql") + "......");
        //       Class.forName(props.getProperty("jdbc.className"));
        conn = DriverManager.getConnection(
        props.getProperty("jdbc.url"),
        props.getProperty("jdbc.user"),
        props.getProperty("jdbc.password"));
        stmt = conn.createStatement();
        rs = stmt.executeQuery(props.getProperty("jdbc.reload.sql"));
        
        while(rs.next()) {
            String theWord = rs.getString("word");
            logger.info("[==========]正在加载Mysql自定义IK扩展词库词条: " + theWord);
            _MainDict.fillSegment(theWord.trim().toCharArray());
        }
    
        Thread.sleep(Integer.valueOf(String.valueOf(props.get("jdbc.reload.interval"))) * 1000);
    } catch (Exception e) {
        logger.error("erorr", e);
    } finally {
        if(rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
                logger.error("error", e);
            }
        }
        if(stmt != null) {
            try {
                stmt.close();
            } catch (SQLException e) {
                logger.error("error", e);
            }
        }
        if(conn != null) {
            try {
                conn.close();
            } catch (SQLException e) {
                logger.error("error", e);
            }
        }
    }
}
复制

/**
* 从MySql中加载远程停用词库
*/
private void loadMySQLStopwordDict() {
    Connection conn = null;
    Statement stmt = null;
    ResultSet rs = null;
    
    try {
        Path file = PathUtils.get(getDictRoot(), "jdbc-reload.properties");
        props.load(new FileInputStream(file.toFile()));
        
        logger.info("[==========]jdbc-reload.properties");
        for(Object key : props.keySet()) {
            logger.info("[==========]" + key + "=" + props.getProperty(String.valueOf(key)));
        }
        logger.info("[==========]query hot stopword dict from mysql, " + props.getProperty("jdbc.reload.stopword.sql") + "......");
        //       Class.forName(props.getProperty("jdbc.className"));
        conn = DriverManager.getConnection(
        props.getProperty("jdbc.url"),
        props.getProperty("jdbc.user"),
        props.getProperty("jdbc.password"));
        stmt = conn.createStatement();
        rs = stmt.executeQuery(props.getProperty("jdbc.reload.stopword.sql"));
        
        while(rs.next()) {
            String theWord = rs.getString("word");
            logger.info("[==========]正在加载Mysql自定义IK停用词库词条: " + theWord);
            _StopWords.fillSegment(theWord.trim().toCharArray());
        }
        Thread.sleep(Integer.valueOf(String.valueOf(props.get("jdbc.reload.interval"))) * 1000);
    } catch (Exception e) {
        logger.error("erorr", e);
    } finally {
        if(rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
                logger.error("error", e);
            }
        }
        if(stmt != null) {
            try {
                stmt.close();
            } catch (SQLException e) {
                logger.error("error", e);
            }
        }
        if(conn != null) {
            try {
                conn.close();
            } catch (SQLException e) {
                logger.error("error", e);
            }
        }
    }
}
复制

完成上述步骤后，拿到elasticsearch-analysis-ik-7.8.0.zip插件，我们将其放在ES安装目录下的plugins目录下，新建一个ik文件夹，将其解压到ik文件夹下；目录结构大概如下，记得要有MySql驱动mysql-connector-java.jar

permission java.net.SocketPermission "*", "connect,resolve";
复制