暂无图片
暂无图片
暂无图片
暂无图片
暂无图片
首页 / Yarn资源池的动态配置

Yarn资源池的动态配置

碧茂大数据 2022-09-13
834

更多精彩，请点击上方蓝字关注我们！

1 获取CM的API接口


  • API的版本为v19



  • 要用到的API接口说明

http://<cm_server>:7180/clusters
# 第一个接口用于获取集群的信息，如集群的名称，以供2、3接口使用
http://<cm_server>:7180/api/v19/clusters/{clusterName}/services/{serviceName}/config
# 第二个接口用于设置Yarn的资源池，接口中的{serviceName}修改为yarn
http://<cm_server>:7180/api/v19/clusters/{clusterName}/commands/poolsRefresh
# 第三个接口用于刷新Yarn的资源池

2 Java

  • pom.xml内容

<dependencies>
    <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpclient</artifactId>
        <version>4.5.4</version>
    </dependency>
    <dependency>
        <groupId>net.sf.json-lib</groupId>
        <artifactId>json-lib</artifactId>
        <version>2.4</version>
        <classifier>jdk15</classifier>
    </dependency>
</dependencies>

  • HttpClient方式调用API接口

package com.cloudera.utils;

import org.apache.commons.lang.StringEscapeUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.net.URI;
import java.util.Map;

public class HttpUtils {

    /**
     * Get方式用户名和密码认证
     * @param url
     * @param headers
     * @param username
     * @param password
     * @return
     */
    public static String getAccessByAuth(String url, Map<String, String> headers, String username, String password) {
        String result = null;

        URI uri = URI.create(url);
        CredentialsProvider credsProvider = new BasicCredentialsProvider();

        credsProvider.setCredentials(new AuthScope(uri.getHost(), uri.getPort()),
                new UsernamePasswordCredentials(username, password));

        CloseableHttpClient httpClient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
                .build();

        HttpGet httpGet = new HttpGet(uri);
        if(headers != null && headers.size() > 0){
            headers.forEach((K,V)->httpGet.addHeader(K,V));
        }

        HttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);
            HttpEntity resultEntity = response.getEntity();
            result = EntityUtils.toString(resultEntity);

            return result;
        } catch (IOException e) {
            e.printStackTrace();
        }


        return null;
    }

    /**
     * Post方式用户名和密码认证
     * @param url
     * @param headers
     * @param data
     * @param username
     * @param password
     * @return
     */
    public static  String postAccessByAuth(String url, Map<String, String> headers, String data, String username, String password) {
        String result = null;

        URI uri = URI.create(url);
        CredentialsProvider credsProvider = new BasicCredentialsProvider();

        credsProvider.setCredentials(new AuthScope(uri.getHost(), uri.getPort()),
                new UsernamePasswordCredentials(username, password));

        CloseableHttpClient httpClient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .build();

        HttpPost post = new HttpPost(uri);

        if(headers != null && headers.size() > 0){
            headers.forEach((K,V)->post.addHeader(K,V));
        }

        try {
            if(data != null) {
                StringEntity entity = new StringEntity(data);
                entity.setContentEncoding("UTF-8");
                entity.setContentType("application/json");
                post.setEntity(entity);
            }

            HttpResponse response = httpClient.execute(post);
            HttpEntity resultEntity = response.getEntity();
            result = EntityUtils.toString(resultEntity);

            return result;
        } catch (Exception e) {
            e.printStackTrace();
        }

        return result;
    }

    /**
     * Put方式用户名和密码认证方式
     * @param url
     * @param headers
     * @param data
     * @param username
     * @param password
     * @return
     */
    public static  String putAccessByAuth(String url, Map<String, String> headers, String data, String username, String password) {
        String result = null;

        URI uri = URI.create(url);
        CredentialsProvider credsProvider = new BasicCredentialsProvider();

        credsProvider.setCredentials(new AuthScope(uri.getHost(), uri.getPort()),
                new UsernamePasswordCredentials(username, password));

        CloseableHttpClient httpClient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .build();

        HttpPut put = new HttpPut(uri);

        if(headers != null && headers.size() > 0){
            headers.forEach((K,V)->put.addHeader(K,V));
        }

        try {
            if(data != null) {
                StringEntity entity = new StringEntity(data);
                entity.setContentEncoding("UTF-8");
                entity.setContentType("application/json");
                put.setEntity(entity);
            }

            HttpResponse response = httpClient.execute(put);
            HttpEntity resultEntity = response.getEntity();
            result = EntityUtils.toString(resultEntity);

            return result;
        } catch (Exception e) {
            e.printStackTrace();
        }

        return result;
    }

}

  • 调用API接口设置资源池

package com.cloudera;

import com.cloudera.api.model.ApiConfig;
import com.cloudera.utils.HttpUtils;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringEscapeUtils;

import java.util.HashMap;

public class RestApiConfPool {

    private static String REQ_CLUSTER_URL = "http://cdh01.fayson.com:7180/api/v19/clusters";
    private static String REQ_SETPOOL_URL = "http://cdh01.fayson.com:7180/api/v19/clusters/cluster/services/yarn/config";
    private static String REQ_FRESHPOOL_URL = "http://cdh01.fayson.com:7180/api/v19/clusters/cluster/commands/poolsRefresh";
    private static String USERNAME = "admin";
    private static String PASSWORD = "admin";

    public static void main(String[] args) {

        HashMap<String, String> headers = new HashMap<>();
        headers.put("Content-Type", "application/json");
        headers.put("Accept", "application/json");

        //获取CM管理的Cluster集群名称
        String result = HttpUtils.getAccessByAuth(REQ_CLUSTER_URL, headers, USERNAME, PASSWORD);
        System.out.println("获取集群信息：" + result);

        //定义资源池配置的JSON配置
        JSONObject requestJson = new JSONObject();
        JSONArray jsonArray = new JSONArray();
        JSONObject yarnjson = new JSONObject();
        String pool_conf = "{\\\"defaultFairSharePreemptionThreshold\\\":null,\\\"defaultFairSharePreemptionTimeout\\\":null,\\\"defaultMinSharePreemptionTimeout\\\":null,\\\"defaultQueueSchedulingPolicy\\\":\\\"fair\\\",\\\"queueMaxAMShareDefault\\\":null,\\\"queueMaxAppsDefault\\\":null,\\\"queuePlacementRules\\\":[{\\\"create\\\":true,\\\"name\\\":\\\"specified\\\",\\\"queue\\\":null,\\\"rules\\\":null},{\\\"create\\\":null,\\\"name\\\":\\\"nestedUserQueue\\\",\\\"queue\\\":null,\\\"rules\\\":[{\\\"create\\\":true,\\\"name\\\":\\\"default\\\",\\\"queue\\\":\\\"users\\\",\\\"rules\\\":null}]},{\\\"create\\\":null,\\\"name\\\":\\\"default\\\",\\\"queue\\\":null,\\\"rules\\\":null}],\\\"queues\\\":[{\\\"aclAdministerApps\\\":\\\"*\\\",\\\"aclSubmitApps\\\":\\\"*\\\",\\\"allowPreemptionFrom\\\":null,\\\"fairSharePreemptionThreshold\\\":null,\\\"fairSharePreemptionTimeout\\\":null,\\\"minSharePreemptionTimeout\\\":null,\\\"name\\\":\\\"root\\\",\\\"queues\\\":[{\\\"aclAdministerApps\\\":null,\\\"aclSubmitApps\\\":null,\\\"allowPreemptionFrom\\\":null,\\\"fairSharePreemptionThreshold\\\":null,\\\"fairSharePreemptionTimeout\\\":null,\\\"minSharePreemptionTimeout\\\":null,\\\"name\\\":\\\"default\\\",\\\"queues\\\":[],\\\"schedulablePropertiesList\\\":[{\\\"impalaDefaultQueryMemLimit\\\":null,\\\"impalaDefaultQueryOptions\\\":null,\\\"impalaMaxMemory\\\":null,\\\"impalaMaxQueuedQueries\\\":null,\\\"impalaMaxRunningQueries\\\":null,\\\"impalaQueueTimeout\\\":null,\\\"maxAMShare\\\":null,\\\"maxChildResources\\\":null,\\\"maxResources\\\":null,\\\"maxRunningApps\\\":null,\\\"minResources\\\":null,\\\"scheduleName\\\":\\\"default\\\",\\\"weight\\\":1.0}],\\\"schedulingPolicy\\\":\\\"drf\\\",\\\"type\\\":null},{\\\"aclAdministerApps\\\":null,\\\"aclSubmitApps\\\":null,\\\"allowPreemptionFrom\\\":null,\\\"fairSharePreemptionThreshold\\\":null,\\\"fairSharePreemptionTimeout\\\":null,\\\"minSharePreemptionTimeout\\\":null,\\\"name\\\":\\\"users\\\",\\\"queues\\\":[],\\\"schedulablePropertiesList\\\":[{\\\"impalaDefaultQueryMemLimit\\\":null,\\\"impalaDefaultQueryOptions\\\":null,\\\"impalaMaxMemory\\\":null,\\\"impalaMaxQueuedQueries\\\":null,\\\"impalaMaxRunningQueries\\\":null,\\\"impalaQueueTimeout\\\":null,\\\"maxAMShare\\\":null,\\\"maxChildResources\\\":null,\\\"maxResources\\\":null,\\\"maxRunningApps\\\":null,\\\"minResources\\\":null,\\\"scheduleName\\\":\\\"default\\\",\\\"weight\\\":2.0}],\\\"schedulingPolicy\\\":\\\"drf\\\",\\\"type\\\":\\\"parent\\\"}],\\\"schedulablePropertiesList\\\":[{\\\"impalaDefaultQueryMemLimit\\\":null,\\\"impalaDefaultQueryOptions\\\":null,\\\"impalaMaxMemory\\\":null,\\\"impalaMaxQueuedQueries\\\":null,\\\"impalaMaxRunningQueries\\\":null,\\\"impalaQueueTimeout\\\":null,\\\"maxAMShare\\\":null,\\\"maxChildResources\\\":null,\\\"maxResources\\\":null,\\\"maxRunningApps\\\":null,\\\"minResources\\\":null,\\\"scheduleName\\\":\\\"default\\\",\\\"weight\\\":1.0}],\\\"schedulingPolicy\\\":\\\"drf\\\",\\\"type\\\":null}],\\\"userMaxAppsDefault\\\":null,\\\"users\\\":[]}";
        yarnjson.put("name", "yarn_fs_scheduled_allocations");
        yarnjson.put("value", pool_conf);
        jsonArray.add(yarnjson);
        requestJson.put("items", jsonArray);
        //注意使用PUT提交，否则会请求失败
        result = HttpUtils.putAccessByAuth(REQ_SETPOOL_URL, headers, StringEscapeUtils.unescapeJava(requestJson.toString()), USERNAME, PASSWORD);

        System.out.println("动态设置Yarn资源池："+ result);

        //刷新资源池
        result = HttpUtils.postAccessByAuth(REQ_FRESHPOOL_URL, headers, null, USERNAME, PASSWORD);

        System.out.println("刷新资源池配置：" + result);
    }
}

3 演示

  • 前Yarn的配置



  • 运行



  • 查看资源池配置



关注公众号：领取精彩视频课程&海量免费语音课程

    yarnstring资源池
    文章转载自碧茂大数据，如果涉嫌侵权，请发送邮件至：contact@modb.pro进行举报，并提供相关证据，一经查实，墨天轮将立刻删除相关内容。

    评论