From c88039d893bc6f67c79aa8b68e6625f47089e56c Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Mon, 18 Jul 2022 15:36:35 +0800 Subject: [PATCH 01/21] fix: remove some unused --- README.md | 4 +- datarangers-sdk-core/pom.xml | 89 +++++++++++++++++++++++++++++++++ datarangers-sdk-starter/pom.xml | 89 +++++++++++++++++++++++++++++++++ pom.xml | 56 +++++++++++++++++++++ 4 files changed, 236 insertions(+), 2 deletions(-) create mode 100644 datarangers-sdk-core/pom.xml create mode 100644 datarangers-sdk-starter/pom.xml create mode 100644 pom.xml diff --git a/README.md b/README.md index ee935b4..db17bf4 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ version是sdk的版本号，当前最新的版本为1.5.3-release。 ### 2. 配置SDK DataRangers SDK需要进行一定的参数配置才能够使用，具体需要配置的参数为： -* domain：datarangers的域名或者ip，支持http和https，例如为 http://www.datarangers.com，在私有化环境中，需要修改为对应的sdk上报域名或者使用DataRangers服务器的ip地址。在saas环境中需要修改成对应的域名： +* domain：datarangers的域名或者ip，支持http和https，例如为 https://www.xxx.com，在私有化环境中，需要修改为对应的sdk上报域名或者使用DataRangers服务器的ip地址。在saas环境中需要修改成对应的域名： * 中国区：https://mcs.ctobsnssdk.com * sg(新加坡): https://mcs.tobsnssdk.com * va(美东): https://mcs.itobsnssdk.com @@ -108,7 +108,7 @@ datarangers.sdk.headers.Host=host # datarangers.sdk.appKeys.${appId}=xxx # 如果是在saas环境中，需要配置openapi, 私有化环境中可以不配置 -# openapi的domain， 国内: https://analytics.volcengineapi.com，国际是: https://datarangers.com +# openapi的domain， 国内: https://analytics.volcengineapi.com，国际是: https://analytics.byteplusapi.com # datarangers.sdk.openapiConfig.domain=xxx # openapi的ak, sk diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml new file mode 100644 index 0000000..dc82031 --- /dev/null +++ b/datarangers-sdk-core/pom.xml @@ -0,0 +1,89 @@ + + + + datarangers-sdk + com.datarangers + 1.5.3-release + + 4.0.0 + jar + datarangers-server-sdk-core + A SDK for datarangers user + https://github.com/volcengine/datarangers-sdk-java + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + datarangers-opensource + datarangers-opensource@bytedance.com + DataFinder + https://www.volcengine.com/product/datafinder + + + com.datarangers + datarangers-sdk-core + 1.5.3-release + + + + + org.apache.httpcomponents.client5 + httpclient5 + 5.0.1 + + + com.fasterxml.jackson.core + jackson-core + 2.11.4 + + + com.fasterxml.jackson.core + jackson-annotations + 2.11.4 + + + com.fasterxml.jackson.core + jackson-databind + 2.11.4 + + + org.apache.kafka + kafka-clients + 0.10.2.1 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + UTF-8 + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + + + + + \ No newline at end of file diff --git a/datarangers-sdk-starter/pom.xml b/datarangers-sdk-starter/pom.xml new file mode 100644 index 0000000..5ac4d7d --- /dev/null +++ b/datarangers-sdk-starter/pom.xml @@ -0,0 +1,89 @@ + + + 4.0.0 + 1.5.3-release + jar + com.datarangers + datarangers-sdk-starter + datarangers-server-sdk-starter + A SDK for datarangers user + https://github.com/volcengine/datarangers-sdk-java + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + + com.datarangers + datarangers-sdk-core + ${project.version} + + + org.springframework.boot + spring-boot-autoconfigure + 2.3.4.RELEASE + + + org.apache.httpcomponents.client5 + httpclient5 + 5.0.1 + + + org.springframework.boot + spring-boot-starter-test + 2.3.4.RELEASE + test + + + log4j-to-slf4j + org.apache.logging.log4j + + + + + org.springframework.boot + spring-boot-test + 2.3.4.RELEASE + test + + + log4j-to-slf4j + org.apache.logging.log4j + 2.15.0 + test + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + UTF-8 + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..874a472 --- /dev/null +++ b/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + + 1.5.3-release + + + com.datarangers + datarangers-sdk + pom + 1.5.3-release + + datarangers-sdk-core + datarangers-sdk-starter + + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + UTF-8 + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + + + + From 28f4bbd04c6ccebcdffa66ceedbe7ed2d49fb656 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Mon, 18 Jul 2022 15:36:35 +0800 Subject: [PATCH 02/21] fix: remove some unused --- README.md | 4 +- datarangers-sdk-core/pom.xml | 89 +++++++++++++++++++++++++++++++++ datarangers-sdk-starter/pom.xml | 89 +++++++++++++++++++++++++++++++++ pom.xml | 56 +++++++++++++++++++++ 4 files changed, 236 insertions(+), 2 deletions(-) create mode 100644 datarangers-sdk-core/pom.xml create mode 100644 datarangers-sdk-starter/pom.xml create mode 100644 pom.xml diff --git a/README.md b/README.md index ee935b4..db17bf4 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ version是sdk的版本号，当前最新的版本为1.5.3-release。 ### 2. 配置SDK DataRangers SDK需要进行一定的参数配置才能够使用，具体需要配置的参数为： -* domain：datarangers的域名或者ip，支持http和https，例如为 http://www.datarangers.com，在私有化环境中，需要修改为对应的sdk上报域名或者使用DataRangers服务器的ip地址。在saas环境中需要修改成对应的域名： +* domain：datarangers的域名或者ip，支持http和https，例如为 https://www.xxx.com，在私有化环境中，需要修改为对应的sdk上报域名或者使用DataRangers服务器的ip地址。在saas环境中需要修改成对应的域名： * 中国区：https://mcs.ctobsnssdk.com * sg(新加坡): https://mcs.tobsnssdk.com * va(美东): https://mcs.itobsnssdk.com @@ -108,7 +108,7 @@ datarangers.sdk.headers.Host=host # datarangers.sdk.appKeys.${appId}=xxx # 如果是在saas环境中，需要配置openapi, 私有化环境中可以不配置 -# openapi的domain， 国内: https://analytics.volcengineapi.com，国际是: https://datarangers.com +# openapi的domain， 国内: https://analytics.volcengineapi.com，国际是: https://analytics.byteplusapi.com # datarangers.sdk.openapiConfig.domain=xxx # openapi的ak, sk diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml new file mode 100644 index 0000000..dc82031 --- /dev/null +++ b/datarangers-sdk-core/pom.xml @@ -0,0 +1,89 @@ + + + + datarangers-sdk + com.datarangers + 1.5.3-release + + 4.0.0 + jar + datarangers-server-sdk-core + A SDK for datarangers user + https://github.com/volcengine/datarangers-sdk-java + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + datarangers-opensource + datarangers-opensource@bytedance.com + DataFinder + https://www.volcengine.com/product/datafinder + + + com.datarangers + datarangers-sdk-core + 1.5.3-release + + + + + org.apache.httpcomponents.client5 + httpclient5 + 5.0.1 + + + com.fasterxml.jackson.core + jackson-core + 2.11.4 + + + com.fasterxml.jackson.core + jackson-annotations + 2.11.4 + + + com.fasterxml.jackson.core + jackson-databind + 2.11.4 + + + org.apache.kafka + kafka-clients + 0.10.2.1 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + UTF-8 + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + + + + + \ No newline at end of file diff --git a/datarangers-sdk-starter/pom.xml b/datarangers-sdk-starter/pom.xml new file mode 100644 index 0000000..5ac4d7d --- /dev/null +++ b/datarangers-sdk-starter/pom.xml @@ -0,0 +1,89 @@ + + + 4.0.0 + 1.5.3-release + jar + com.datarangers + datarangers-sdk-starter + datarangers-server-sdk-starter + A SDK for datarangers user + https://github.com/volcengine/datarangers-sdk-java + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + + com.datarangers + datarangers-sdk-core + ${project.version} + + + org.springframework.boot + spring-boot-autoconfigure + 2.3.4.RELEASE + + + org.apache.httpcomponents.client5 + httpclient5 + 5.0.1 + + + org.springframework.boot + spring-boot-starter-test + 2.3.4.RELEASE + test + + + log4j-to-slf4j + org.apache.logging.log4j + + + + + org.springframework.boot + spring-boot-test + 2.3.4.RELEASE + test + + + log4j-to-slf4j + org.apache.logging.log4j + 2.15.0 + test + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + UTF-8 + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..874a472 --- /dev/null +++ b/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + + 1.5.3-release + + + com.datarangers + datarangers-sdk + pom + 1.5.3-release + + datarangers-sdk-core + datarangers-sdk-starter + + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + UTF-8 + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + + + + From 606a2d1c69998db504147444427ea688f1003241 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Wed, 3 Aug 2022 17:55:36 +0800 Subject: [PATCH 03/21] feature: support batch --- README.md | 19 +- datarangers-sdk-core/pom.xml | 4 +- .../asynccollector/CollectorContainer.java | 4 + .../asynccollector/CollectorQueue.java | 10 + .../datarangers/asynccollector/Consumer.java | 220 ++++++++++-------- .../asynccollector/RangersCollectorQueue.java | 21 ++ .../com/datarangers/collector/Collector.java | 10 +- .../com/datarangers/config/Constants.java | 3 +- .../DataRangersSDKConfigProperties.java | 28 +++ .../com/datarangers/config/EventConfig.java | 10 + .../java/com/datarangers/sender/Callback.java | 17 +- .../com/datarangers/sender/MessageSender.java | 11 + .../sender/PrivatizationMessageSender.java | 10 + .../sender/callback/LoggingCallback.java | 27 ++- .../java/com/datarangers/util/HttpUtils.java | 17 +- datarangers-sdk-starter/pom.xml | 2 +- pom.xml | 4 +- 17 files changed, 291 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index db17bf4..5f8036f 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ datarangers-sdk-java是 [DataFinder](https://www.volcengine.com/product/datafind com.datarangers datarangers-sdk-core - 1.5.3-release + 1.5.4-release ``` -version是sdk的版本号，当前最新的版本为1.5.3-release。 +version是sdk的版本号，当前最新的版本为1.5.4-release。 火山引擎仓库地址： ```xml @@ -86,7 +86,7 @@ DataRangers SDK需要进行一定的参数配置才能够使用，具体需要 com.datarangers datarangers-sdk-starter - 1.5.3-release + 1.5.4-release ``` @@ -115,13 +115,24 @@ datarangers.sdk.headers.Host=host # datarangers.sdk.openapiConfig.ak=xxx # datarangers.sdk.openapiConfig.sk=xxx - # 是否保存到本地,如果需要配合logagent使用需要将其定义为true datarangers.sdk.save=true # 异步方式的发送线程数量,如果为logagent模式请设置为1 datarangers.sdk.threadCount=4 +# 异步方式的发送核心线程数量，建议corePoolSize 跟threadCount 配置成一样 +datarangers.sdk.corePoolSize=4 # 异步方式队列长度 datarangers.sdk.queueSize=102400 + +# 是否使用批量发送,默认为false +#datarangers.sdk.sendBatch=true + +# 批量发送的大小 +#datarangers.sdk.batchSize=16 + +# 批量的等待时间，当批量达到batchSize，或者等待时间超过waitTimeMs，就立刻发送 +#datarangers.sdk.waitTimeMs=100 + # 保存日志文件路径 datarangers.sdk.eventSavePath=logs/ # 保存日志文件名 diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml index dc82031..6896b6a 100644 --- a/datarangers-sdk-core/pom.xml +++ b/datarangers-sdk-core/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - 1.5.3-release + 1.5.4-release 4.0.0 jar @@ -28,7 +28,7 @@ com.datarangers datarangers-sdk-core - 1.5.3-release + 1.5.4-release diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorContainer.java b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorContainer.java index 6fd53ae..c8f96c6 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorContainer.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorContainer.java @@ -43,6 +43,10 @@ public List consume(int waitTimeMs) throws InterruptedException { return handleMessage(messageQueue.poll(waitTimeMs)); } + public List consume(int size, int waitTimeMs) throws InterruptedException { + return handleMessage(messageQueue.poll(size, waitTimeMs)); + } + public int size() { return messageQueue.size(); } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorQueue.java b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorQueue.java index 8edf846..bbae533 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorQueue.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorQueue.java @@ -23,7 +23,17 @@ public interface CollectorQueue { * @date: 2021/2/7 15:51 */ List take() throws InterruptedException; + List poll(int waitTimeMs) throws InterruptedException; + + /** + * 每次等待waitTimeMs，获取数据最多size的数据 + * @param size + * @param waitTimeMs + * @return + * @throws InterruptedException + */ + List poll(int size, int waitTimeMs) throws InterruptedException; /** * 功能描述: 发送一个Message到队列中 * diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java index 6d1f198..5961d95 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java @@ -13,125 +13,143 @@ import com.datarangers.message.AppMessage; import com.datarangers.message.Message; import com.datarangers.sender.MessageSenderFactory; - -import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + public class Consumer implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(Consumer.class); - private static RangersLoggerWriterPool pool; - private CollectorContainer collectorContainer; - private DataRangersSDKConfigProperties sdkConfigProperties; - - public Consumer(CollectorContainer collectorContainer, - DataRangersSDKConfigProperties sdkConfigProperties) { - this.collectorContainer = collectorContainer; - this.sdkConfigProperties = sdkConfigProperties; - } - - public static void setWriterPool(final List targetPrefixes, String targetNames, - int maxSize) { - if (pool == null) { - synchronized (Consumer.class) { + private static final Logger logger = LoggerFactory.getLogger(Consumer.class); + private static RangersLoggerWriterPool pool; + private CollectorContainer collectorContainer; + private DataRangersSDKConfigProperties sdkConfigProperties; + + public Consumer(CollectorContainer collectorContainer, + DataRangersSDKConfigProperties sdkConfigProperties) { + this.collectorContainer = collectorContainer; + this.sdkConfigProperties = sdkConfigProperties; + } + + public static void setWriterPool(final List targetPrefixes, String targetNames, + int maxSize) { if (pool == null) { - pool = RangersLoggerWriterPool.getInstance(targetPrefixes, targetNames, maxSize); + synchronized (Consumer.class) { + if (pool == null) { + pool = RangersLoggerWriterPool.getInstance(targetPrefixes, targetNames, maxSize); + } + } } - } } - } - - private void send() throws Exception { - while (true) { - try { - List messages = collectorContainer.consume(); - if (messages != null) { - messages.forEach(message -> { - doSend(message); - }); + + private void send() throws Exception { + while (true) { + try { + List messages = collectorContainer.consume(); + if (messages != null) { + messages.forEach(message -> { + doSend(message); + }); + } + } catch (Throwable e) { + e.printStackTrace(); + logger.error("consumer send error", e); + } } - } catch (Throwable e) { - e.printStackTrace(); - logger.error("consumer send error", e); - } } - } - - private void write() throws Exception { - while (true) { - try { - List messages = collectorContainer.consume(); - if (messages != null) { - messages.forEach(message -> { - doWrite(message); - }); + + private void sendBatch() throws Exception { + while (true) { + try { + List messages = collectorContainer.consume(sdkConfigProperties.getBatchSize(), + sdkConfigProperties.getWaitTimeMs()); + if (messages != null && messages.size() > 0) { + MessageSenderFactory.getMessageSender(messages.get(0)) + .sendBatch(messages, this.sdkConfigProperties); + } + } catch (Throwable e) { + e.printStackTrace(); + logger.error("consumer send error", e); + } } - } catch (Throwable e) { - logger.error("consumer write error", e); - } + } + private void write() throws Exception { + while (true) { + try { + List messages = collectorContainer.consume(); + if (messages != null) { + messages.forEach(message -> { + doWrite(message); + }); + } + } catch (Throwable e) { + logger.error("consumer write error", e); + } + + } } - } - - @Override - public void run() { - try { - if (EventConfig.saveFlag) { - write(); - } else { - send(); - } - } catch (Exception e) { - e.printStackTrace(); - logger.error("consumer run error", e); + + @Override + public void run() { + try { + if (EventConfig.saveFlag) { + write(); + } else if (sdkConfigProperties.isSendBatch()) { + sendBatch(); + } else { + send(); + } + } catch (Exception e) { + e.printStackTrace(); + logger.error("consumer run error", e); + } } - } - - public void flush() { - try { - System.out.println("flush message start"); - logger.info("flush message start"); - int count = 0; - if (collectorContainer.getMessageQueue() != null) { - Message message = collectorContainer.getMessageQueue().poll(); - while (message != null) { - count++; - message = collectorContainer.handleMessage(message); - if (EventConfig.saveFlag) { + + public void flush() { + try { + System.out.println("flush message start"); + logger.info("flush message start"); + int count = 0; + if (collectorContainer.getMessageQueue() != null) { + Message message = collectorContainer.getMessageQueue().poll(); + while (message != null) { + count++; + message = collectorContainer.handleMessage(message); + if (EventConfig.saveFlag) { + doWrite(message); + } else { + doSend(message); + } + message = collectorContainer.getMessageQueue().poll(); + } + } + + logger.info("flush message success. size: {}", count); + System.out.println("flush message success. size: " + count); + } catch (Exception e) { + e.printStackTrace(); + logger.error("flush message error", e); + } + } + + public void flush(Message message) { + message = collectorContainer.handleMessage(message); + if (EventConfig.saveFlag) { doWrite(message); - } else { + } else { doSend(message); - } - message = collectorContainer.getMessageQueue().poll(); } - } + } - logger.info("flush message success. size: {}", count); - System.out.println("flush message success. size: " + count); - } catch (Exception e) { - e.printStackTrace(); - logger.error("flush message error", e); + private void doSend(Message message) { + MessageSenderFactory.getMessageSender(message) + .send(message, this.sdkConfigProperties); } - } - - public void flush(Message message) { - message = collectorContainer.handleMessage(message); - if (EventConfig.saveFlag) { - doWrite(message); - } else { - doSend(message); + + private void doWrite(Message message) { + AppMessage appMessage = message.getAppMessage(); + pool.getWriter(appMessage.getUserUniqueId()) + .write(RangersJSONConfig.getInstance().toJson(appMessage) + "\n"); } - } - - private void doSend(Message message) { - MessageSenderFactory.getMessageSender(message) - .send(message, this.sdkConfigProperties); - } - - private void doWrite(Message message) { - AppMessage appMessage = message.getAppMessage(); - pool.getWriter(appMessage.getUserUniqueId()) - .write(RangersJSONConfig.getInstance().toJson(appMessage) + "\n"); - } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/RangersCollectorQueue.java b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/RangersCollectorQueue.java index deced6c..39793a8 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/RangersCollectorQueue.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/RangersCollectorQueue.java @@ -2,6 +2,7 @@ import com.datarangers.message.Message; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -57,6 +58,26 @@ public List poll(int waitTimeMs) throws InterruptedException { return null; } + @Override + public List poll(int size, int waitTimeMs) throws InterruptedException { + List messages = new ArrayList<>(); + Message msg = queue.poll(waitTimeMs, TimeUnit.MILLISECONDS); + if(msg != null){ + messages.add(msg); + } + + // 只有 + while(messages.size() < size){ + msg = queue.poll(waitTimeMs, TimeUnit.MILLISECONDS); + if(msg == null){ + // 退出循环 + break; + } + messages.add(msg); + } + return messages; + } + @Override public void put(Message t) throws InterruptedException { queue.put(t); diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java index fc69e3a..9a9e60d 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java @@ -119,14 +119,14 @@ private void sendByKafka(AppMessage appMessage) { public void onCompletion(RecordMetadata recordMetadata, Exception e) { if (e != null) { logger.error(String.format("kafka send message error. value: \r\n %s", sendMessage), e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e)); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); } } }); } catch (Exception e) { e.printStackTrace(); logger.error(String.format("kafka send message error. value: \r\n %s", sendMessage), e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e)); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); } } @@ -136,7 +136,7 @@ private void syncSendMessage(Message message, String sendMessage) { } catch (Exception e) { e.printStackTrace(); logger.error("sync send message error", e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e)); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); } } @@ -147,11 +147,11 @@ private void asyncSendMessage(Message message, String sendMessage) { } catch (Exception e) { e.printStackTrace(); logger.error("async send message error", e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e)); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); } } else { logger.error("getMessageQueue is null"); - getCallback().onFailed(new FailedData(sendMessage, "getMessageQueue is null")); + getCallback().onFailed(new FailedData(sendMessage, "getMessageQueue is null", false)); } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java index 8dae554..cc4ac23 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java @@ -11,10 +11,11 @@ import java.util.GregorianCalendar; public class Constants { - public static final String SDK_VERSION = "datarangers_sdk_1.5.3-release"; + public static final String SDK_VERSION = "datarangers_sdk_1.5.4-release"; public static DateTimeFormatter FULL_HOUR = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH"); public static DateTimeFormatter FULL_DAY = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); public static final String APP_LOG_PATH = "/sdk/log"; + public static final String APP_LIST_PATH = "/sdk/list"; public static final String DEFAULT_USER = "__rangers"; public static final String INIT_ERROR = "sdk config must not be null"; diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java index ada2c0b..751fd7b 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java @@ -58,6 +58,34 @@ public class DataRangersSDKConfigProperties { public int queueSize = 10240; public boolean send = true; + private boolean sendBatch = false; + private int batchSize = 20; + private int waitTimeMs = 100; + + public boolean isSendBatch() { + return sendBatch; + } + + public void setSendBatch(boolean sendBatch) { + this.sendBatch = sendBatch; + } + + public int getBatchSize() { + return batchSize; + } + + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } + + public int getWaitTimeMs() { + return waitTimeMs; + } + + public void setWaitTimeMs(int waitTimeMs) { + this.waitTimeMs = waitTimeMs; + } + public boolean isEnable() { return enable; } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java index 7f6e555..9244bfe 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java @@ -12,6 +12,7 @@ public class EventConfig { public static String appUrl; + public static String appListUrl; public static Header[] headers; public static Map SEND_HEADER; public static boolean saveFlag = false; @@ -19,6 +20,7 @@ public class EventConfig { public static void setUrl(String url) { setAppUrl(url + Constants.APP_LOG_PATH); + setAppListUrl(url + Constants.APP_LIST_PATH); } public static String getAppUrl() { @@ -28,4 +30,12 @@ public static String getAppUrl() { public static void setAppUrl(String appUrl) { EventConfig.appUrl = appUrl; } + + public static String getAppListUrl() { + return EventConfig.appListUrl; + } + + public static void setAppListUrl(String appUrl) { + EventConfig.appListUrl = appUrl; + } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/sender/Callback.java b/datarangers-sdk-core/src/main/java/com/datarangers/sender/Callback.java index 824662f..2628a59 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/sender/Callback.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/sender/Callback.java @@ -13,14 +13,15 @@ public interface Callback { class FailedData { - public FailedData(String message, String cause) { - this(message, cause, null); + public FailedData(String message, String cause, boolean listable) { + this(message, cause, null, listable); } - public FailedData(String message, String cause, Exception exception) { + public FailedData(String message, String cause, Exception exception, boolean listable) { this.message = message; this.cause = cause; this.exception = exception; + this.listable = listable; } /** @@ -38,6 +39,8 @@ public FailedData(String message, String cause, Exception exception) { */ private Exception exception; + private boolean listable; + public String getMessage() { return message; } @@ -61,5 +64,13 @@ public Exception getException() { public void setException(Exception exception) { this.exception = exception; } + + public boolean isListable() { + return listable; + } + + public void setListable(boolean listable) { + this.listable = listable; + } } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/sender/MessageSender.java b/datarangers-sdk-core/src/main/java/com/datarangers/sender/MessageSender.java index 2305771..53f3f00 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/sender/MessageSender.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/sender/MessageSender.java @@ -3,6 +3,8 @@ import com.datarangers.config.DataRangersSDKConfigProperties; import com.datarangers.message.Message; +import java.util.List; + /** * @Author zhangpeng.spin@bytedance.com * @Date 2021-07-22 @@ -15,4 +17,13 @@ public interface MessageSender { * @param sdkConfigProperties */ void send(Message message, DataRangersSDKConfigProperties sdkConfigProperties); + + /** + * 使用批量上报 + * @param message + * @param sdkConfigProperties + */ + default void sendBatch(List message, DataRangersSDKConfigProperties sdkConfigProperties) { + throw new UnsupportedOperationException("Not support batch"); + }; } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/sender/PrivatizationMessageSender.java b/datarangers-sdk-core/src/main/java/com/datarangers/sender/PrivatizationMessageSender.java index 89f1396..c69156e 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/sender/PrivatizationMessageSender.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/sender/PrivatizationMessageSender.java @@ -3,9 +3,13 @@ import com.datarangers.config.DataRangersSDKConfigProperties; import com.datarangers.config.EventConfig; import com.datarangers.config.RangersJSONConfig; +import com.datarangers.message.AppMessage; import com.datarangers.message.Message; import com.datarangers.util.HttpUtils; +import java.util.List; +import java.util.stream.Collectors; + /** * @Author zhangpeng.spin@bytedance.com * @Date 2021-07-22 @@ -17,4 +21,10 @@ public void send(Message message, DataRangersSDKConfigProperties sdkConfigProper Object sendMessage = message.getAppMessage(); HttpUtils.post(EventConfig.getAppUrl(), RangersJSONConfig.getInstance().toJson(sendMessage), EventConfig.SEND_HEADER); } + + @Override + public void sendBatch(List message, DataRangersSDKConfigProperties sdkConfigProperties) { + List sendMessages = message.stream().map(n -> n.getAppMessage()).collect(Collectors.toList()); + HttpUtils.post(EventConfig.getAppListUrl(), RangersJSONConfig.getInstance().toJson(sendMessages), EventConfig.SEND_HEADER); + } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/sender/callback/LoggingCallback.java b/datarangers-sdk-core/src/main/java/com/datarangers/sender/callback/LoggingCallback.java index 5e90356..bc66da4 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/sender/callback/LoggingCallback.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/sender/callback/LoggingCallback.java @@ -1,7 +1,13 @@ package com.datarangers.sender.callback; +import com.datarangers.config.RangersJSONConfig; import com.datarangers.logger.RangersLoggerWriter; import com.datarangers.sender.Callback; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; /** * @Author zhangpeng.spin@bytedance.com @@ -9,6 +15,8 @@ */ public class LoggingCallback implements Callback { + public static final Logger logger = LoggerFactory.getLogger(LoggingCallback.class); + private static RangersLoggerWriter writer; private static final Object lock = new Object(); @@ -19,7 +27,24 @@ public LoggingCallback(final String targetPrefix, final String targetName, int m @Override public void onFailed(FailedData failedData) { - writeFailedMessage(failedData.getMessage()); + if (failedData.getMessage() == null) { + return; + } + if(!failedData.isListable()){ + writeFailedMessage(failedData.getMessage()); + }else{ + try { + List list = RangersJSONConfig.getInstance().fromJson(failedData.getMessage(), List.class); + list.forEach(n-> { + writeFailedMessage(RangersJSONConfig.getInstance().toJson(n)); + }); + } catch (IOException e) { + e.printStackTrace(); + logger.error("json error", e); + writeFailedMessage(failedData.getMessage()); + } + } + } private void writeFailedMessage(String message) { diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/util/HttpUtils.java b/datarangers-sdk-core/src/main/java/com/datarangers/util/HttpUtils.java index 279f681..8e8ded5 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/util/HttpUtils.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/util/HttpUtils.java @@ -7,9 +7,7 @@ package com.datarangers.util; import com.datarangers.config.HttpConfig; -import com.datarangers.config.EventConfig; import com.datarangers.config.RangersJSONConfig; -import com.datarangers.logger.RangersLoggerWriter; import com.datarangers.sender.Callback; import com.datarangers.sender.Callback.FailedData; @@ -27,7 +25,6 @@ import javax.net.ssl.X509TrustManager; import org.apache.hc.client5.http.classic.HttpClient; -import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy; @@ -202,12 +199,12 @@ public static void request(String method, String url, String body, Map 2) { logger.error(String.format("request error, io error: %s, resultStr: %s", e.getMessage(), resultStr), e); - callback.onFailed(new FailedData(body, e.toString(), e)); + callback.onFailed(new FailedData(body, e.toString(), e, isListable(url))); } else { count++; post(url, body, headers, count); @@ -217,7 +214,7 @@ public static void request(String method, String url, String body, Map 4.0.0 - 1.5.3-release + 1.5.4-release jar com.datarangers datarangers-sdk-starter diff --git a/pom.xml b/pom.xml index 874a472..4de759c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,13 +4,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 1.5.3-release + 1.5.4-release com.datarangers datarangers-sdk pom - 1.5.3-release + 1.5.4-release datarangers-sdk-core datarangers-sdk-starter From 172305cd4fb9c229e5f852122fbc2c27ecb8ac5c Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Wed, 3 Aug 2022 17:55:36 +0800 Subject: [PATCH 04/21] feature: support batch --- README.md | 19 +- datarangers-sdk-core/pom.xml | 4 +- .../asynccollector/CollectorContainer.java | 4 + .../asynccollector/CollectorQueue.java | 10 + .../datarangers/asynccollector/Consumer.java | 220 ++++++++++-------- .../asynccollector/RangersCollectorQueue.java | 21 ++ .../com/datarangers/collector/Collector.java | 10 +- .../com/datarangers/config/Constants.java | 3 +- .../DataRangersSDKConfigProperties.java | 28 +++ .../com/datarangers/config/EventConfig.java | 10 + .../java/com/datarangers/sender/Callback.java | 17 +- .../com/datarangers/sender/MessageSender.java | 11 + .../sender/PrivatizationMessageSender.java | 10 + .../sender/callback/LoggingCallback.java | 27 ++- .../java/com/datarangers/util/HttpUtils.java | 17 +- datarangers-sdk-starter/pom.xml | 2 +- pom.xml | 4 +- 17 files changed, 291 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index db17bf4..5f8036f 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ datarangers-sdk-java是 [DataFinder](https://www.volcengine.com/product/datafind com.datarangers datarangers-sdk-core - 1.5.3-release + 1.5.4-release ``` -version是sdk的版本号，当前最新的版本为1.5.3-release。 +version是sdk的版本号，当前最新的版本为1.5.4-release。 火山引擎仓库地址： ```xml @@ -86,7 +86,7 @@ DataRangers SDK需要进行一定的参数配置才能够使用，具体需要 com.datarangers datarangers-sdk-starter - 1.5.3-release + 1.5.4-release ``` @@ -115,13 +115,24 @@ datarangers.sdk.headers.Host=host # datarangers.sdk.openapiConfig.ak=xxx # datarangers.sdk.openapiConfig.sk=xxx - # 是否保存到本地,如果需要配合logagent使用需要将其定义为true datarangers.sdk.save=true # 异步方式的发送线程数量,如果为logagent模式请设置为1 datarangers.sdk.threadCount=4 +# 异步方式的发送核心线程数量，建议corePoolSize 跟threadCount 配置成一样 +datarangers.sdk.corePoolSize=4 # 异步方式队列长度 datarangers.sdk.queueSize=102400 + +# 是否使用批量发送,默认为false +#datarangers.sdk.sendBatch=true + +# 批量发送的大小 +#datarangers.sdk.batchSize=16 + +# 批量的等待时间，当批量达到batchSize，或者等待时间超过waitTimeMs，就立刻发送 +#datarangers.sdk.waitTimeMs=100 + # 保存日志文件路径 datarangers.sdk.eventSavePath=logs/ # 保存日志文件名 diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml index dc82031..6896b6a 100644 --- a/datarangers-sdk-core/pom.xml +++ b/datarangers-sdk-core/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - 1.5.3-release + 1.5.4-release 4.0.0 jar @@ -28,7 +28,7 @@ com.datarangers datarangers-sdk-core - 1.5.3-release + 1.5.4-release diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorContainer.java b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorContainer.java index 6fd53ae..c8f96c6 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorContainer.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorContainer.java @@ -43,6 +43,10 @@ public List consume(int waitTimeMs) throws InterruptedException { return handleMessage(messageQueue.poll(waitTimeMs)); } + public List consume(int size, int waitTimeMs) throws InterruptedException { + return handleMessage(messageQueue.poll(size, waitTimeMs)); + } + public int size() { return messageQueue.size(); } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorQueue.java b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorQueue.java index 8edf846..bbae533 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorQueue.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorQueue.java @@ -23,7 +23,17 @@ public interface CollectorQueue { * @date: 2021/2/7 15:51 */ List take() throws InterruptedException; + List poll(int waitTimeMs) throws InterruptedException; + + /** + * 每次等待waitTimeMs，获取数据最多size的数据 + * @param size + * @param waitTimeMs + * @return + * @throws InterruptedException + */ + List poll(int size, int waitTimeMs) throws InterruptedException; /** * 功能描述: 发送一个Message到队列中 * diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java index 6d1f198..5961d95 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java @@ -13,125 +13,143 @@ import com.datarangers.message.AppMessage; import com.datarangers.message.Message; import com.datarangers.sender.MessageSenderFactory; - -import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + public class Consumer implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(Consumer.class); - private static RangersLoggerWriterPool pool; - private CollectorContainer collectorContainer; - private DataRangersSDKConfigProperties sdkConfigProperties; - - public Consumer(CollectorContainer collectorContainer, - DataRangersSDKConfigProperties sdkConfigProperties) { - this.collectorContainer = collectorContainer; - this.sdkConfigProperties = sdkConfigProperties; - } - - public static void setWriterPool(final List targetPrefixes, String targetNames, - int maxSize) { - if (pool == null) { - synchronized (Consumer.class) { + private static final Logger logger = LoggerFactory.getLogger(Consumer.class); + private static RangersLoggerWriterPool pool; + private CollectorContainer collectorContainer; + private DataRangersSDKConfigProperties sdkConfigProperties; + + public Consumer(CollectorContainer collectorContainer, + DataRangersSDKConfigProperties sdkConfigProperties) { + this.collectorContainer = collectorContainer; + this.sdkConfigProperties = sdkConfigProperties; + } + + public static void setWriterPool(final List targetPrefixes, String targetNames, + int maxSize) { if (pool == null) { - pool = RangersLoggerWriterPool.getInstance(targetPrefixes, targetNames, maxSize); + synchronized (Consumer.class) { + if (pool == null) { + pool = RangersLoggerWriterPool.getInstance(targetPrefixes, targetNames, maxSize); + } + } } - } } - } - - private void send() throws Exception { - while (true) { - try { - List messages = collectorContainer.consume(); - if (messages != null) { - messages.forEach(message -> { - doSend(message); - }); + + private void send() throws Exception { + while (true) { + try { + List messages = collectorContainer.consume(); + if (messages != null) { + messages.forEach(message -> { + doSend(message); + }); + } + } catch (Throwable e) { + e.printStackTrace(); + logger.error("consumer send error", e); + } } - } catch (Throwable e) { - e.printStackTrace(); - logger.error("consumer send error", e); - } } - } - - private void write() throws Exception { - while (true) { - try { - List messages = collectorContainer.consume(); - if (messages != null) { - messages.forEach(message -> { - doWrite(message); - }); + + private void sendBatch() throws Exception { + while (true) { + try { + List messages = collectorContainer.consume(sdkConfigProperties.getBatchSize(), + sdkConfigProperties.getWaitTimeMs()); + if (messages != null && messages.size() > 0) { + MessageSenderFactory.getMessageSender(messages.get(0)) + .sendBatch(messages, this.sdkConfigProperties); + } + } catch (Throwable e) { + e.printStackTrace(); + logger.error("consumer send error", e); + } } - } catch (Throwable e) { - logger.error("consumer write error", e); - } + } + private void write() throws Exception { + while (true) { + try { + List messages = collectorContainer.consume(); + if (messages != null) { + messages.forEach(message -> { + doWrite(message); + }); + } + } catch (Throwable e) { + logger.error("consumer write error", e); + } + + } } - } - - @Override - public void run() { - try { - if (EventConfig.saveFlag) { - write(); - } else { - send(); - } - } catch (Exception e) { - e.printStackTrace(); - logger.error("consumer run error", e); + + @Override + public void run() { + try { + if (EventConfig.saveFlag) { + write(); + } else if (sdkConfigProperties.isSendBatch()) { + sendBatch(); + } else { + send(); + } + } catch (Exception e) { + e.printStackTrace(); + logger.error("consumer run error", e); + } } - } - - public void flush() { - try { - System.out.println("flush message start"); - logger.info("flush message start"); - int count = 0; - if (collectorContainer.getMessageQueue() != null) { - Message message = collectorContainer.getMessageQueue().poll(); - while (message != null) { - count++; - message = collectorContainer.handleMessage(message); - if (EventConfig.saveFlag) { + + public void flush() { + try { + System.out.println("flush message start"); + logger.info("flush message start"); + int count = 0; + if (collectorContainer.getMessageQueue() != null) { + Message message = collectorContainer.getMessageQueue().poll(); + while (message != null) { + count++; + message = collectorContainer.handleMessage(message); + if (EventConfig.saveFlag) { + doWrite(message); + } else { + doSend(message); + } + message = collectorContainer.getMessageQueue().poll(); + } + } + + logger.info("flush message success. size: {}", count); + System.out.println("flush message success. size: " + count); + } catch (Exception e) { + e.printStackTrace(); + logger.error("flush message error", e); + } + } + + public void flush(Message message) { + message = collectorContainer.handleMessage(message); + if (EventConfig.saveFlag) { doWrite(message); - } else { + } else { doSend(message); - } - message = collectorContainer.getMessageQueue().poll(); } - } + } - logger.info("flush message success. size: {}", count); - System.out.println("flush message success. size: " + count); - } catch (Exception e) { - e.printStackTrace(); - logger.error("flush message error", e); + private void doSend(Message message) { + MessageSenderFactory.getMessageSender(message) + .send(message, this.sdkConfigProperties); } - } - - public void flush(Message message) { - message = collectorContainer.handleMessage(message); - if (EventConfig.saveFlag) { - doWrite(message); - } else { - doSend(message); + + private void doWrite(Message message) { + AppMessage appMessage = message.getAppMessage(); + pool.getWriter(appMessage.getUserUniqueId()) + .write(RangersJSONConfig.getInstance().toJson(appMessage) + "\n"); } - } - - private void doSend(Message message) { - MessageSenderFactory.getMessageSender(message) - .send(message, this.sdkConfigProperties); - } - - private void doWrite(Message message) { - AppMessage appMessage = message.getAppMessage(); - pool.getWriter(appMessage.getUserUniqueId()) - .write(RangersJSONConfig.getInstance().toJson(appMessage) + "\n"); - } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/RangersCollectorQueue.java b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/RangersCollectorQueue.java index deced6c..39793a8 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/RangersCollectorQueue.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/RangersCollectorQueue.java @@ -2,6 +2,7 @@ import com.datarangers.message.Message; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -57,6 +58,26 @@ public List poll(int waitTimeMs) throws InterruptedException { return null; } + @Override + public List poll(int size, int waitTimeMs) throws InterruptedException { + List messages = new ArrayList<>(); + Message msg = queue.poll(waitTimeMs, TimeUnit.MILLISECONDS); + if(msg != null){ + messages.add(msg); + } + + // 只有 + while(messages.size() < size){ + msg = queue.poll(waitTimeMs, TimeUnit.MILLISECONDS); + if(msg == null){ + // 退出循环 + break; + } + messages.add(msg); + } + return messages; + } + @Override public void put(Message t) throws InterruptedException { queue.put(t); diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java index fc69e3a..9a9e60d 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java @@ -119,14 +119,14 @@ private void sendByKafka(AppMessage appMessage) { public void onCompletion(RecordMetadata recordMetadata, Exception e) { if (e != null) { logger.error(String.format("kafka send message error. value: \r\n %s", sendMessage), e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e)); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); } } }); } catch (Exception e) { e.printStackTrace(); logger.error(String.format("kafka send message error. value: \r\n %s", sendMessage), e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e)); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); } } @@ -136,7 +136,7 @@ private void syncSendMessage(Message message, String sendMessage) { } catch (Exception e) { e.printStackTrace(); logger.error("sync send message error", e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e)); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); } } @@ -147,11 +147,11 @@ private void asyncSendMessage(Message message, String sendMessage) { } catch (Exception e) { e.printStackTrace(); logger.error("async send message error", e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e)); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); } } else { logger.error("getMessageQueue is null"); - getCallback().onFailed(new FailedData(sendMessage, "getMessageQueue is null")); + getCallback().onFailed(new FailedData(sendMessage, "getMessageQueue is null", false)); } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java index 8dae554..cc4ac23 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java @@ -11,10 +11,11 @@ import java.util.GregorianCalendar; public class Constants { - public static final String SDK_VERSION = "datarangers_sdk_1.5.3-release"; + public static final String SDK_VERSION = "datarangers_sdk_1.5.4-release"; public static DateTimeFormatter FULL_HOUR = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH"); public static DateTimeFormatter FULL_DAY = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); public static final String APP_LOG_PATH = "/sdk/log"; + public static final String APP_LIST_PATH = "/sdk/list"; public static final String DEFAULT_USER = "__rangers"; public static final String INIT_ERROR = "sdk config must not be null"; diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java index ada2c0b..751fd7b 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java @@ -58,6 +58,34 @@ public class DataRangersSDKConfigProperties { public int queueSize = 10240; public boolean send = true; + private boolean sendBatch = false; + private int batchSize = 20; + private int waitTimeMs = 100; + + public boolean isSendBatch() { + return sendBatch; + } + + public void setSendBatch(boolean sendBatch) { + this.sendBatch = sendBatch; + } + + public int getBatchSize() { + return batchSize; + } + + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } + + public int getWaitTimeMs() { + return waitTimeMs; + } + + public void setWaitTimeMs(int waitTimeMs) { + this.waitTimeMs = waitTimeMs; + } + public boolean isEnable() { return enable; } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java index 7f6e555..9244bfe 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java @@ -12,6 +12,7 @@ public class EventConfig { public static String appUrl; + public static String appListUrl; public static Header[] headers; public static Map SEND_HEADER; public static boolean saveFlag = false; @@ -19,6 +20,7 @@ public class EventConfig { public static void setUrl(String url) { setAppUrl(url + Constants.APP_LOG_PATH); + setAppListUrl(url + Constants.APP_LIST_PATH); } public static String getAppUrl() { @@ -28,4 +30,12 @@ public static String getAppUrl() { public static void setAppUrl(String appUrl) { EventConfig.appUrl = appUrl; } + + public static String getAppListUrl() { + return EventConfig.appListUrl; + } + + public static void setAppListUrl(String appUrl) { + EventConfig.appListUrl = appUrl; + } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/sender/Callback.java b/datarangers-sdk-core/src/main/java/com/datarangers/sender/Callback.java index 824662f..2628a59 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/sender/Callback.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/sender/Callback.java @@ -13,14 +13,15 @@ public interface Callback { class FailedData { - public FailedData(String message, String cause) { - this(message, cause, null); + public FailedData(String message, String cause, boolean listable) { + this(message, cause, null, listable); } - public FailedData(String message, String cause, Exception exception) { + public FailedData(String message, String cause, Exception exception, boolean listable) { this.message = message; this.cause = cause; this.exception = exception; + this.listable = listable; } /** @@ -38,6 +39,8 @@ public FailedData(String message, String cause, Exception exception) { */ private Exception exception; + private boolean listable; + public String getMessage() { return message; } @@ -61,5 +64,13 @@ public Exception getException() { public void setException(Exception exception) { this.exception = exception; } + + public boolean isListable() { + return listable; + } + + public void setListable(boolean listable) { + this.listable = listable; + } } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/sender/MessageSender.java b/datarangers-sdk-core/src/main/java/com/datarangers/sender/MessageSender.java index 2305771..53f3f00 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/sender/MessageSender.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/sender/MessageSender.java @@ -3,6 +3,8 @@ import com.datarangers.config.DataRangersSDKConfigProperties; import com.datarangers.message.Message; +import java.util.List; + /** * @Author zhangpeng.spin@bytedance.com * @Date 2021-07-22 @@ -15,4 +17,13 @@ public interface MessageSender { * @param sdkConfigProperties */ void send(Message message, DataRangersSDKConfigProperties sdkConfigProperties); + + /** + * 使用批量上报 + * @param message + * @param sdkConfigProperties + */ + default void sendBatch(List message, DataRangersSDKConfigProperties sdkConfigProperties) { + throw new UnsupportedOperationException("Not support batch"); + }; } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/sender/PrivatizationMessageSender.java b/datarangers-sdk-core/src/main/java/com/datarangers/sender/PrivatizationMessageSender.java index 89f1396..c69156e 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/sender/PrivatizationMessageSender.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/sender/PrivatizationMessageSender.java @@ -3,9 +3,13 @@ import com.datarangers.config.DataRangersSDKConfigProperties; import com.datarangers.config.EventConfig; import com.datarangers.config.RangersJSONConfig; +import com.datarangers.message.AppMessage; import com.datarangers.message.Message; import com.datarangers.util.HttpUtils; +import java.util.List; +import java.util.stream.Collectors; + /** * @Author zhangpeng.spin@bytedance.com * @Date 2021-07-22 @@ -17,4 +21,10 @@ public void send(Message message, DataRangersSDKConfigProperties sdkConfigProper Object sendMessage = message.getAppMessage(); HttpUtils.post(EventConfig.getAppUrl(), RangersJSONConfig.getInstance().toJson(sendMessage), EventConfig.SEND_HEADER); } + + @Override + public void sendBatch(List message, DataRangersSDKConfigProperties sdkConfigProperties) { + List sendMessages = message.stream().map(n -> n.getAppMessage()).collect(Collectors.toList()); + HttpUtils.post(EventConfig.getAppListUrl(), RangersJSONConfig.getInstance().toJson(sendMessages), EventConfig.SEND_HEADER); + } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/sender/callback/LoggingCallback.java b/datarangers-sdk-core/src/main/java/com/datarangers/sender/callback/LoggingCallback.java index 5e90356..bc66da4 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/sender/callback/LoggingCallback.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/sender/callback/LoggingCallback.java @@ -1,7 +1,13 @@ package com.datarangers.sender.callback; +import com.datarangers.config.RangersJSONConfig; import com.datarangers.logger.RangersLoggerWriter; import com.datarangers.sender.Callback; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; /** * @Author zhangpeng.spin@bytedance.com @@ -9,6 +15,8 @@ */ public class LoggingCallback implements Callback { + public static final Logger logger = LoggerFactory.getLogger(LoggingCallback.class); + private static RangersLoggerWriter writer; private static final Object lock = new Object(); @@ -19,7 +27,24 @@ public LoggingCallback(final String targetPrefix, final String targetName, int m @Override public void onFailed(FailedData failedData) { - writeFailedMessage(failedData.getMessage()); + if (failedData.getMessage() == null) { + return; + } + if(!failedData.isListable()){ + writeFailedMessage(failedData.getMessage()); + }else{ + try { + List list = RangersJSONConfig.getInstance().fromJson(failedData.getMessage(), List.class); + list.forEach(n-> { + writeFailedMessage(RangersJSONConfig.getInstance().toJson(n)); + }); + } catch (IOException e) { + e.printStackTrace(); + logger.error("json error", e); + writeFailedMessage(failedData.getMessage()); + } + } + } private void writeFailedMessage(String message) { diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/util/HttpUtils.java b/datarangers-sdk-core/src/main/java/com/datarangers/util/HttpUtils.java index 279f681..8e8ded5 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/util/HttpUtils.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/util/HttpUtils.java @@ -7,9 +7,7 @@ package com.datarangers.util; import com.datarangers.config.HttpConfig; -import com.datarangers.config.EventConfig; import com.datarangers.config.RangersJSONConfig; -import com.datarangers.logger.RangersLoggerWriter; import com.datarangers.sender.Callback; import com.datarangers.sender.Callback.FailedData; @@ -27,7 +25,6 @@ import javax.net.ssl.X509TrustManager; import org.apache.hc.client5.http.classic.HttpClient; -import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy; @@ -202,12 +199,12 @@ public static void request(String method, String url, String body, Map 2) { logger.error(String.format("request error, io error: %s, resultStr: %s", e.getMessage(), resultStr), e); - callback.onFailed(new FailedData(body, e.toString(), e)); + callback.onFailed(new FailedData(body, e.toString(), e, isListable(url))); } else { count++; post(url, body, headers, count); @@ -217,7 +214,7 @@ public static void request(String method, String url, String body, Map 4.0.0 - 1.5.3-release + 1.5.4-release jar com.datarangers datarangers-sdk-starter diff --git a/pom.xml b/pom.xml index 874a472..4de759c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,13 +4,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 1.5.3-release + 1.5.4-release com.datarangers datarangers-sdk pom - 1.5.3-release + 1.5.4-release datarangers-sdk-core datarangers-sdk-starter From a3ed9e918ad1afe77f1872464d312e6472298514 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Mon, 8 Aug 2022 21:44:43 +0800 Subject: [PATCH 05/21] fix: fix write log bug --- README.md | 6 +++--- datarangers-sdk-core/pom.xml | 4 ++-- .../src/main/java/com/datarangers/config/Constants.java | 2 +- .../java/com/datarangers/logger/RangersLoggerWriter.java | 4 ++-- datarangers-sdk-starter/pom.xml | 2 +- pom.xml | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5f8036f..fad4f13 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ datarangers-sdk-java是 [DataFinder](https://www.volcengine.com/product/datafind com.datarangers datarangers-sdk-core - 1.5.4-release + 1.5.5-release ``` -version是sdk的版本号，当前最新的版本为1.5.4-release。 +version是sdk的版本号，当前最新的版本为1.5.5-release。 火山引擎仓库地址： ```xml @@ -86,7 +86,7 @@ DataRangers SDK需要进行一定的参数配置才能够使用，具体需要 com.datarangers datarangers-sdk-starter - 1.5.4-release + 1.5.5-release ``` diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml index 6896b6a..00c4021 100644 --- a/datarangers-sdk-core/pom.xml +++ b/datarangers-sdk-core/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - 1.5.4-release + 1.5.5-release 4.0.0 jar @@ -28,7 +28,7 @@ com.datarangers datarangers-sdk-core - 1.5.4-release + 1.5.5-release diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java index cc4ac23..add87dc 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java @@ -11,7 +11,7 @@ import java.util.GregorianCalendar; public class Constants { - public static final String SDK_VERSION = "datarangers_sdk_1.5.4-release"; + public static final String SDK_VERSION = "datarangers_sdk_1.5.5-release"; public static DateTimeFormatter FULL_HOUR = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH"); public static DateTimeFormatter FULL_DAY = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); public static final String APP_LOG_PATH = "/sdk/log"; diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/logger/RangersLoggerWriter.java b/datarangers-sdk-core/src/main/java/com/datarangers/logger/RangersLoggerWriter.java index 07fca3b..d3b1614 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/logger/RangersLoggerWriter.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/logger/RangersLoggerWriter.java @@ -101,7 +101,7 @@ public RangersLoggerWriter(final String targetPrefix, final String targetName, i File parent = new File(targetPrefix); if (!parent.exists()) parent.mkdirs(); this.currentIndex = setCurrentIndex(); - if (this.currentIndex == 0) this.currentIndex++; + this.currentIndex++; fullTarget = this.targetPrefix + "/" + this.targetName; this.output = new File(fullTarget); currentName = targetName + "." + LocalDateTime.now().format(Constants.FULL_HOUR); @@ -120,7 +120,7 @@ public RangersLoggerWriter(final String targetPrefix, final String targetName) { private int setCurrentIndex() { String current = LocalDateTime.now().format(Constants.FULL_HOUR); - String full = targetName + "-" + current + "-"; + String full = targetName + "." + current + "."; int number = 0; for (File f : new File(targetPrefix).listFiles()) { if (f.getName().contains(full)) { diff --git a/datarangers-sdk-starter/pom.xml b/datarangers-sdk-starter/pom.xml index b9f232b..2af176a 100644 --- a/datarangers-sdk-starter/pom.xml +++ b/datarangers-sdk-starter/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 1.5.4-release + 1.5.5-release jar com.datarangers datarangers-sdk-starter diff --git a/pom.xml b/pom.xml index 4de759c..6c43407 100644 --- a/pom.xml +++ b/pom.xml @@ -4,13 +4,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 1.5.4-release + 1.5.5-release com.datarangers datarangers-sdk pom - 1.5.4-release + 1.5.5-release datarangers-sdk-core datarangers-sdk-starter From c83a579e677d7b7b8c79ef52a6369abddebcae59 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Mon, 8 Aug 2022 21:44:43 +0800 Subject: [PATCH 06/21] fix: fix write log bug --- README.md | 6 +++--- datarangers-sdk-core/pom.xml | 4 ++-- .../src/main/java/com/datarangers/config/Constants.java | 2 +- .../java/com/datarangers/logger/RangersLoggerWriter.java | 4 ++-- datarangers-sdk-starter/pom.xml | 2 +- pom.xml | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5f8036f..fad4f13 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ datarangers-sdk-java是 [DataFinder](https://www.volcengine.com/product/datafind com.datarangers datarangers-sdk-core - 1.5.4-release + 1.5.5-release ``` -version是sdk的版本号，当前最新的版本为1.5.4-release。 +version是sdk的版本号，当前最新的版本为1.5.5-release。 火山引擎仓库地址： ```xml @@ -86,7 +86,7 @@ DataRangers SDK需要进行一定的参数配置才能够使用，具体需要 com.datarangers datarangers-sdk-starter - 1.5.4-release + 1.5.5-release ``` diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml index 6896b6a..00c4021 100644 --- a/datarangers-sdk-core/pom.xml +++ b/datarangers-sdk-core/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - 1.5.4-release + 1.5.5-release 4.0.0 jar @@ -28,7 +28,7 @@ com.datarangers datarangers-sdk-core - 1.5.4-release + 1.5.5-release diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java index cc4ac23..add87dc 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java @@ -11,7 +11,7 @@ import java.util.GregorianCalendar; public class Constants { - public static final String SDK_VERSION = "datarangers_sdk_1.5.4-release"; + public static final String SDK_VERSION = "datarangers_sdk_1.5.5-release"; public static DateTimeFormatter FULL_HOUR = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH"); public static DateTimeFormatter FULL_DAY = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); public static final String APP_LOG_PATH = "/sdk/log"; diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/logger/RangersLoggerWriter.java b/datarangers-sdk-core/src/main/java/com/datarangers/logger/RangersLoggerWriter.java index 07fca3b..d3b1614 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/logger/RangersLoggerWriter.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/logger/RangersLoggerWriter.java @@ -101,7 +101,7 @@ public RangersLoggerWriter(final String targetPrefix, final String targetName, i File parent = new File(targetPrefix); if (!parent.exists()) parent.mkdirs(); this.currentIndex = setCurrentIndex(); - if (this.currentIndex == 0) this.currentIndex++; + this.currentIndex++; fullTarget = this.targetPrefix + "/" + this.targetName; this.output = new File(fullTarget); currentName = targetName + "." + LocalDateTime.now().format(Constants.FULL_HOUR); @@ -120,7 +120,7 @@ public RangersLoggerWriter(final String targetPrefix, final String targetName) { private int setCurrentIndex() { String current = LocalDateTime.now().format(Constants.FULL_HOUR); - String full = targetName + "-" + current + "-"; + String full = targetName + "." + current + "."; int number = 0; for (File f : new File(targetPrefix).listFiles()) { if (f.getName().contains(full)) { diff --git a/datarangers-sdk-starter/pom.xml b/datarangers-sdk-starter/pom.xml index b9f232b..2af176a 100644 --- a/datarangers-sdk-starter/pom.xml +++ b/datarangers-sdk-starter/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 1.5.4-release + 1.5.5-release jar com.datarangers datarangers-sdk-starter diff --git a/pom.xml b/pom.xml index 4de759c..6c43407 100644 --- a/pom.xml +++ b/pom.xml @@ -4,13 +4,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 1.5.4-release + 1.5.5-release com.datarangers datarangers-sdk pom - 1.5.4-release + 1.5.5-release datarangers-sdk-core datarangers-sdk-starter From d0868a1c92afec554cb18bc0254b8b64dd9bb969 Mon Sep 17 00:00:00 2001 From: qifeng Date: Mon, 22 Aug 2022 16:04:27 +0800 Subject: [PATCH 07/21] perf: config, log --- datarangers-sdk-core/pom.xml | 6 + .../asynccollector/CollectorCounter.java | 2 +- .../collector/AppEventCollector.java | 10 +- .../com/datarangers/collector/Collector.java | 472 ++++++++++++------ .../DataRangersSDKConfigProperties.java | 230 ++------- .../com/datarangers/config/HttpConfig.java | 9 + .../com/datarangers/config/KafkaConfig.java | 21 + .../main/java/test/AppEventCollectorTest.java | 38 ++ 8 files changed, 435 insertions(+), 353 deletions(-) create mode 100644 datarangers-sdk-core/src/main/java/test/AppEventCollectorTest.java diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml index 00c4021..e3e96e7 100644 --- a/datarangers-sdk-core/pom.xml +++ b/datarangers-sdk-core/pom.xml @@ -57,6 +57,12 @@ kafka-clients 0.10.2.1 + + junit + junit + 4.12 + compile + diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorCounter.java b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorCounter.java index c3eed78..493c5c2 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorCounter.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/CollectorCounter.java @@ -34,7 +34,7 @@ public void run() { FileOutputStream stream = null; try { stream = new FileOutputStream(output, true); - Map status = new HashMap() {{ + Map status = new HashMap(2) {{ put("history", CollectorContainer.SEND_HISTORY); put("queue_length", Collector.collectorContainer.size()); }}; diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/collector/AppEventCollector.java b/datarangers-sdk-core/src/main/java/com/datarangers/collector/AppEventCollector.java index 3cf9e12..927d4d0 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/collector/AppEventCollector.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/collector/AppEventCollector.java @@ -32,15 +32,6 @@ public AppEventCollector(String appType, DataRangersSDKConfigProperties properti public AppEventCollector(String appType, DataRangersSDKConfigProperties properties, Callback cb) { super(appType, properties, cb); - if (properties != null) { - properties.init(); - - // 设置同步发送的consumer，队列满的时候使用 - setConsumer(new Consumer(Collector.collectorContainer, this.properties)); - - } else { - System.out.println(Constants.INIT_ERROR); - } } @Override @@ -227,6 +218,7 @@ private void sendEvents(Header header, List events, MessageType messageTy appMessage.setAppType(getAppType()); appMessage.setHeader(header); appMessage.addEvents(events); + appMessage.setTraceId(UUID.randomUUID().toString()); message.setAppMessage(appMessage); send(message); diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java index 9a9e60d..6b38a0d 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java @@ -6,185 +6,343 @@ */ package com.datarangers.collector; -import com.datarangers.asynccollector.CollectorContainer; -import com.datarangers.asynccollector.Consumer; +import com.datarangers.asynccollector.*; import com.datarangers.config.*; +import com.datarangers.logger.RangersFileCleaner; import com.datarangers.message.AppMessage; import com.datarangers.message.Message; import com.datarangers.message.MessageEnv; import com.datarangers.sender.Callback; import com.datarangers.sender.Callback.FailedData; +import com.datarangers.sender.callback.LoggingCallback; +import com.datarangers.util.HttpUtils; +import org.apache.hc.client5.http.classic.HttpClient; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.message.BasicHeader; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; /** * @author taojian */ public abstract class Collector implements EventCollector { - private String appType; - public static final Logger logger = LoggerFactory.getLogger("DatarangersLog"); - public static ExecutorService httpRequestPool = null; - public static ScheduledExecutorService scheduled = null; - public static CollectorContainer collectorContainer; - - private boolean enable; - protected DataRangersSDKConfigProperties properties; - protected Callback callback; - protected Consumer consumer = null; - protected KafkaProducer kafkaProducer; - - public Collector(String appType, DataRangersSDKConfigProperties properties, Callback cb) { - this.appType = appType; - this.enable = properties.isEnable(); - this.properties = properties; - this.callback = cb; - this.properties.setCallback(this.getCallback()); - this.initKafkaProducer(); - } - - private void initKafkaProducer(){ - if(SdkMode.KAFKA != this.properties.getMode()){ - return; - } - // 设置过了就不需要再自己创建 - if(kafkaProducer != null){ - return; - } - kafkaProducer = createProducer(this.properties.getKafka()); - } - - private KafkaProducer createProducer(KafkaConfig kafkaConfig) { - Properties props = new Properties(); - props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); - props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); - props.put("bootstrap.servers", kafkaConfig.getBootstrapServers()); - Map map = kafkaConfig.getProperties(); - if (map != null && (!map.isEmpty())) { - props.putAll(map); - } - return new KafkaProducer<>(props); - } - public String getAppType() { - return appType; - } - - public Collector setAppType(String appType) { - this.appType = appType; - return this; - } - - - public void send(Message message) { - sendMessage(message); - } - - protected void sendMessage(Message message) { - if (!enable) { - return; - } - message.merge(); - String sendMessage; - - validate(message); - if(kafkaProducer != null){ - // 使用kafka的方式 - sendByKafka(message.getAppMessage()); - return; - } - sendMessage = RangersJSONConfig.getInstance().toJson(message.getAppMessage()); - if (this.properties.isSync()) { - syncSendMessage(message, sendMessage); - } else { - asyncSendMessage(message, sendMessage); - } - } - - private void sendByKafka(AppMessage appMessage) { - // kafka sender，header 添加固定的头 - appMessage.getHeader().setSource(Constants.SDK_SERVER); - String sendMessage = RangersJSONConfig.getInstance().toJson(appMessage); - try { - ProducerRecord producerRecord = new ProducerRecord<>(properties.getKafka().getTopic(), sendMessage); - kafkaProducer.send(producerRecord, new org.apache.kafka.clients.producer.Callback() { - @Override - public void onCompletion(RecordMetadata recordMetadata, Exception e) { - if (e != null) { + private String appType; + public static final Logger logger = LoggerFactory.getLogger("DatarangersLog"); + public static ExecutorService httpRequestPool = null; + public static ScheduledExecutorService scheduled = null; + public static CollectorContainer collectorContainer; + private boolean enable; + protected DataRangersSDKConfigProperties properties; + protected Callback callback; + protected Consumer consumer = null; + protected KafkaProducer kafkaProducer; + private static volatile Boolean IS_INIT = false; + + public Collector(String appType, DataRangersSDKConfigProperties properties, Callback cb) { + logger.info("sdk config properties: {}", properties); + System.out.println("sdk config properties: " + properties.toString()); + this.appType = appType; + this.enable = properties.isEnable(); + this.properties = properties; + this.callback = cb; + this.properties.setCallback(this.getCallback()); + this.init(); + } + + public String getAppType() { + return appType; + } + + public void setAppType(String appType) { + this.appType = appType; + } + + public Callback getCallback() { + return callback; + } + + public void setCallback(Callback callback) { + this.callback = callback; + } + + public Consumer getConsumer() { + return consumer; + } + + public void setConsumer(Consumer consumer) { + this.consumer = consumer; + } + + + private void initKafkaProducer() { + logger.info("init kafka producer"); + if (SdkMode.KAFKA != this.properties.getMode()) { + return; + } + // 设置过了就不需要再自己创建 + if (kafkaProducer != null) { + return; + } + kafkaProducer = createProducer(this.properties.getKafka()); + } + + private KafkaProducer createProducer(KafkaConfig kafkaConfig) { + Properties props = new Properties(); + props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + props.put("bootstrap.servers", kafkaConfig.getBootstrapServers()); + Map map = kafkaConfig.getProperties(); + if (map != null && (!map.isEmpty())) { + props.putAll(map); + } + return new KafkaProducer<>(props); + } + + public void send(Message message) { + sendMessage(message); + } + + private void sendMessage(Message message) { + if (!enable) { + return; + } + message.merge(); + String sendMessage; + + validate(message); + if (kafkaProducer != null) { + // 使用kafka的方式 + sendByKafka(message.getAppMessage()); + return; + } + sendMessage = RangersJSONConfig.getInstance().toJson(message.getAppMessage()); + if (this.properties.isSync()) { + syncSendMessage(message, sendMessage); + } else { + asyncSendMessage(message, sendMessage); + } + } + + private void sendByKafka(AppMessage appMessage) { + // kafka sender，header 添加固定的头 + appMessage.getHeader().setSource(Constants.SDK_SERVER); + String sendMessage = RangersJSONConfig.getInstance().toJson(appMessage); + try { + ProducerRecord producerRecord = new ProducerRecord<>(properties.getKafka().getTopic(), sendMessage); + kafkaProducer.send(producerRecord, new org.apache.kafka.clients.producer.Callback() { + @Override + public void onCompletion(RecordMetadata recordMetadata, Exception e) { + if (e != null) { + logger.error(String.format("kafka send message error. value: \r\n %s", sendMessage), e); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); + } + } + }); + } catch (Exception e) { + e.printStackTrace(); logger.error(String.format("kafka send message error. value: \r\n %s", sendMessage), e); getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); - } - } - }); - } catch (Exception e) { - e.printStackTrace(); - logger.error(String.format("kafka send message error. value: \r\n %s", sendMessage), e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); - } - } - - private void syncSendMessage(Message message, String sendMessage) { - try { - this.consumer.flush(message); - } catch (Exception e) { - e.printStackTrace(); - logger.error("sync send message error", e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); - } - } - - private void asyncSendMessage(Message message, String sendMessage) { - if (collectorContainer.getMessageQueue() != null) { - try { - collectorContainer.produce(message); - } catch (Exception e) { - e.printStackTrace(); - logger.error("async send message error", e); - getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); - } - } else { - logger.error("getMessageQueue is null"); - getCallback().onFailed(new FailedData(sendMessage, "getMessageQueue is null", false)); - } - } - - /** - * message 检查 - */ - private void validate(Message message) { - // 当前只有saas需要校验下appkey - if (this.properties.getMessageEnv() != MessageEnv.SAAS) { - return; - } - - Integer appId = message.getAppMessage().getAppId(); - Map appKeys = this.properties.getAppKeys(); - String appKey = appKeys.get(appId); - if (appKey == null) { - throw new IllegalArgumentException("App key cannot be empty. app_id: " + appId); - } - } - - public Callback getCallback() { - return callback; - } - - public void setCallback(Callback callback) { - this.callback = callback; - } - - public Consumer getConsumer() { - return consumer; - } - - public void setConsumer(Consumer consumer) { - this.consumer = consumer; - } + } + } + + private void syncSendMessage(Message message, String sendMessage) { + try { + this.consumer.flush(message); + } catch (Exception e) { + e.printStackTrace(); + logger.error("sync send message error", e); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); + } + } + + private void asyncSendMessage(Message message, String sendMessage) { + if (collectorContainer.getMessageQueue() != null) { + try { + collectorContainer.produce(message); + } catch (Exception e) { + e.printStackTrace(); + logger.error("async send message error", e); + getCallback().onFailed(new FailedData(sendMessage, e.getMessage(), e, false)); + } + } else { + logger.error("getMessageQueue is null"); + getCallback().onFailed(new FailedData(sendMessage, "getMessageQueue is null", false)); + } + } + + /** + * message 检查 + */ + private void validate(Message message) { + // 当前只有saas需要校验下appkey + if (this.properties.getMessageEnv() != MessageEnv.SAAS) { + return; + } + + Integer appId = message.getAppMessage().getAppId(); + Map appKeys = this.properties.getAppKeys(); + String appKey = appKeys.get(appId); + if (appKey == null) { + throw new IllegalArgumentException("App key cannot be empty. app_id: " + appId); + } + } + + /** + * 配置初始化 + */ + private void init() { + if (!IS_INIT) { + synchronized (Collector.class) { + if (!IS_INIT) { + initLogger(); + initCommon(); + initKafkaProducer(); + initConsumer(); + initHook(); + IS_INIT = true; + } + } + } + } + + /** + * 日志pool初始化 + */ + private void initLogger() { + logger.info("init log writer pool"); + List eventFilePaths = properties.getEventFilePaths(); + String eventSaveName = properties.getEventSaveName(); + int eventSaveMaxFileSize = properties.getEventSaveMaxFileSize(); + String eventSavePath = properties.getEventSavePath(); + + Consumer.setWriterPool(eventFilePaths, eventSaveName, eventSaveMaxFileSize); + if (properties.getCallback() == null) { + properties.setCallback(new LoggingCallback(eventSavePath, "error-" + eventSaveName, + eventSaveMaxFileSize)); + } + } + + /** + * eventConfig,httpclient,EventConfig 初始化 + */ + private void initCommon() { + HttpConfig httpConfig = properties.getHttpConfig(); + HttpClient httpClient = properties.getCustomHttpClient(); + Callback callback = properties.getCallback(); + int httpTimeOut = properties.getHttpTimeout(); + + //EventConfig配置 + EventConfig.saveFlag = SdkMode.FILE == properties.getMode(); + EventConfig.sendFlag = SdkMode.HTTP == properties.getMode(); + EventConfig.setUrl(properties.getDomain()); + + if (EventConfig.sendFlag) { + if (httpConfig.getMaxPerRoute() < properties.getCorePoolSize()) { + httpConfig.setMaxPerRoute(properties.getCorePoolSize()); + } + if (httpConfig.getMaxTotal() < httpConfig.getMaxPerRoute()) { + httpConfig.setMaxTotal(httpConfig.getMaxPerRoute()); + } + // 老版本配置做兼容 + httpConfig.initTimeOut(httpTimeOut); + //httpclient 初始化 + HttpUtils.createHttpClient(httpConfig, httpClient, callback); + + //EventConfig 初始化 + if (EventConfig.SEND_HEADER == null) { + EventConfig.SEND_HEADER = properties.getHeaders(); + EventConfig.SEND_HEADER.put("User-Agent", "DataRangers Java SDK"); + EventConfig.SEND_HEADER.put("Content-Type", "application/json"); + List
headerList = new ArrayList<>(); + EventConfig.SEND_HEADER + .forEach((key, value) -> headerList.add(new BasicHeader(key, value))); + EventConfig.headers = headerList.toArray(new Header[0]); + } + } + } + + /** + * 初始化消费者, httpRequestPool, 日志记录和清理任务 + */ + private void initConsumer() { + logger.info("init consumer"); + int threadCount = this.properties.getCorePoolSize(); + CollectorQueue userQueue = this.properties.getUserQueue(); + + if (EventConfig.saveFlag) { + threadCount = 1; + logger.info("Start LogAgent Mode"); + } else { + logger.info("Start Http Mode"); + } + + // 如果客户自定义了queue，则需要替换为客户自定义queue，否则使用默认的队列 + if (userQueue == null) { + collectorContainer = new CollectorContainer( + RangersCollectorQueue.getInstance(this.properties.getQueueSize())); + } else { + collectorContainer = new CollectorContainer(userQueue); + } + + boolean isSync = this.properties.isSync(); + boolean hasConsumer = this.properties.isHasConsumer(); + boolean hasProducer = this.properties.isHasProducer(); + String eventSavePath = this.properties.getEventSavePath(); + List eventFilePaths = properties.getEventFilePaths(); + String eventSaveName = this.properties.getEventSaveName(); + int eventSaveMaxDays = this.properties.getEventSaveMaxDays(); + // 设置同步发送的consumer，队列满的时候使用 + if (isSync) { + setConsumer(new Consumer(Collector.collectorContainer, this.properties)); + } + // 异步起多个消费者 + if (!isSync && hasConsumer && httpRequestPool == null) { + httpRequestPool = Executors.newFixedThreadPool(this.properties.getCorePoolSize()); + for (int i = 0; i < threadCount; i++) { + //必须全部消费同一个队列 + httpRequestPool.execute(new Consumer(collectorContainer, properties)); + } + } + if ((!isSync) && hasProducer) { + //定时记录日志的条数 + scheduled = Executors.newSingleThreadScheduledExecutor(); + scheduled + .scheduleAtFixedRate(new CollectorCounter(eventSavePath), 0, 2, TimeUnit.MINUTES); + if (EventConfig.saveFlag) { + // 清理日志文件定时任务, 每隔12小时清理一次 + scheduled.scheduleAtFixedRate( + new RangersFileCleaner(eventFilePaths, eventSaveName, eventSaveMaxDays), + 0, 12, TimeUnit.HOURS); + logger.info("Start DataRangers Cleaner/Record Thread"); + } + } + } + + /** + * jvm关闭时，一些需要做的清理任务 + */ + private void initHook() { + logger.info("init hook"); + Runtime.getRuntime().addShutdownHook(new Thread( + () -> { + if (Collector.httpRequestPool != null) { + Collector.httpRequestPool.shutdown(); + } + + new Consumer(Collector.collectorContainer, properties).flush(); + })); + } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java index 751fd7b..b02d6a4 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java @@ -7,34 +7,20 @@ package com.datarangers.config; import com.datarangers.asynccollector.*; -import com.datarangers.collector.Collector; -import com.datarangers.logger.RangersFileCleaner; import com.datarangers.message.MessageEnv; import com.datarangers.sender.Callback; -import com.datarangers.sender.callback.LoggingCallback; -import com.datarangers.util.HttpUtils; import org.apache.hc.client5.http.classic.HttpClient; -import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.message.BasicHeader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.time.ZoneOffset; import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; /** * @author hTangle */ public class DataRangersSDKConfigProperties { - public static final Logger logger = LoggerFactory.getLogger(DataRangersSDKConfigProperties.class); public Map headers; public String domain; - public int threadPoolCount = 1; - public int maxPoolSize = 8; public int corePoolSize = 4; /** @@ -54,7 +40,6 @@ public class DataRangersSDKConfigProperties { */ @Deprecated public boolean save = false; - public int threadCount = 1; public int queueSize = 10240; public boolean send = true; @@ -62,45 +47,13 @@ public class DataRangersSDKConfigProperties { private int batchSize = 20; private int waitTimeMs = 100; - public boolean isSendBatch() { - return sendBatch; - } - - public void setSendBatch(boolean sendBatch) { - this.sendBatch = sendBatch; - } - - public int getBatchSize() { - return batchSize; - } - - public void setBatchSize(int batchSize) { - this.batchSize = batchSize; - } - - public int getWaitTimeMs() { - return waitTimeMs; - } - - public void setWaitTimeMs(int waitTimeMs) { - this.waitTimeMs = waitTimeMs; - } - - public boolean isEnable() { - return enable; - } - - public void setEnable(boolean enable) { - this.enable = enable; - } - public boolean enable = true; public String eventSavePath = "logs/"; public List eventFilePaths; public String eventSaveName = "datarangers.log"; - public int eventSaveMaxHistory = 20; public int eventSaveMaxFileSize = 100; + // 日志清理时间 public int eventSaveMaxDays = 5; public CollectorQueue userQueue; @@ -120,8 +73,8 @@ public void setEnable(boolean enable) { private String env = "privatization"; private List SAAS_DOMAIN_URLS = Arrays.asList( - "https://mcs.ctobsnssdk.com", - "https://mcs.tobsnssdk.com"); + "https://mcs.ctobsnssdk.com", + "https://mcs.tobsnssdk.com"); /** * saas openapi 配置地址 @@ -138,6 +91,38 @@ public void setEnable(boolean enable) { private SdkMode mode; private KafkaConfig kafka; + public boolean isSendBatch() { + return sendBatch; + } + + public void setSendBatch(boolean sendBatch) { + this.sendBatch = sendBatch; + } + + public int getBatchSize() { + return batchSize; + } + + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } + + public int getWaitTimeMs() { + return waitTimeMs; + } + + public void setWaitTimeMs(int waitTimeMs) { + this.waitTimeMs = waitTimeMs; + } + + public boolean isEnable() { + return enable; + } + + public void setEnable(boolean enable) { + this.enable = enable; + } + public KafkaConfig getKafka() { return kafka; } @@ -243,15 +228,6 @@ public DataRangersSDKConfigProperties setEventSaveName(String eventSaveName) { return this; } - public int getEventSaveMaxHistory() { - return eventSaveMaxHistory; - } - - public DataRangersSDKConfigProperties setEventSaveMaxHistory(int eventSaveMaxHistory) { - this.eventSaveMaxHistory = eventSaveMaxHistory; - return this; - } - public int getEventSaveMaxFileSize() { return eventSaveMaxFileSize; } @@ -270,14 +246,6 @@ public DataRangersSDKConfigProperties setSend(boolean send) { return this; } - public int getThreadCount() { - return threadCount; - } - - public void setThreadCount(int threadCount) { - this.threadCount = threadCount; - } - public int getQueueSize() { return queueSize; } @@ -286,14 +254,6 @@ public void setQueueSize(int queueSize) { this.queueSize = queueSize; } - public int getMaxPoolSize() { - return maxPoolSize; - } - - public void setMaxPoolSize(int maxPoolSize) { - this.maxPoolSize = maxPoolSize; - } - public int getCorePoolSize() { return corePoolSize; } @@ -321,14 +281,6 @@ public void setDomain(String domain) { this.domain = domain; } - public int getThreadPoolCount() { - return threadPoolCount; - } - - public void setThreadPoolCount(int threadPoolCount) { - this.threadPoolCount = threadPoolCount; - } - public ZoneOffset getTimeOffset() { return timeOffset; } @@ -356,98 +308,6 @@ public void setTimeZone(String timeZone) { this.timeZone = timeZone; } - public void setLogger() { - Consumer.setWriterPool(getEventFilePaths(), getEventSaveName(), getEventSaveMaxFileSize()); - if (callback == null) { - setCallback(new LoggingCallback(getEventSavePath(), "error-" + getEventSaveName(), - getEventSaveMaxFileSize())); - } - } - - public void setCommon() { - EventConfig.saveFlag = SdkMode.FILE == getMode(); - EventConfig.sendFlag = SdkMode.HTTP == getMode(); - if (EventConfig.sendFlag) { - httpConfig = this.getHttpConfig(); - if (httpConfig.getMaxPerRoute() < this.getThreadCount()) { - httpConfig.setMaxPerRoute(this.getThreadCount()); - } - if (httpConfig.getMaxTotal() < httpConfig.getMaxPerRoute()) { - httpConfig.setMaxTotal(httpConfig.getMaxPerRoute()); - } - httpConfig.initTimeOut(getHttpTimeout()); - HttpUtils - .createHttpClient(this.getHttpConfig(), this.getCustomHttpClient(), this.getCallback()); - if (EventConfig.SEND_HEADER == null) { - EventConfig.SEND_HEADER = getHeaders(); - EventConfig.SEND_HEADER.put("User-Agent", "DataRangers Java SDK"); - EventConfig.SEND_HEADER.put("Content-Type", "application/json"); - List
headerList = new ArrayList<>(); - EventConfig.SEND_HEADER - .forEach((key, value) -> headerList.add(new BasicHeader(key, value))); - EventConfig.headers = headerList.toArray(new Header[0]); - } - } - setConsumer(getThreadCount()); - EventConfig.setUrl(getDomain()); - } - - - public ExecutorService setThreadPool() { - return Executors.newFixedThreadPool(getCorePoolSize()); - } - - public void setConsumer(int threadCount) { - if (EventConfig.saveFlag) { - threadCount = 1; - logger.info("Start LogAgent Mode"); - } else { - logger.info("Start Http Mode"); - } - if (userQueue == null) { - Collector.collectorContainer = new CollectorContainer( - RangersCollectorQueue.getInstance(getQueueSize())); - } else { - //如果客户自定义了queue，则需要替换为客户自定义queue - Collector.collectorContainer = new CollectorContainer(userQueue); - } - if ((!sync) && hasConsumer && Collector.httpRequestPool == null) { - //有消费者才初始化消费者 - Collector.httpRequestPool = setThreadPool(); - for (int i = 0; i < threadCount; i++) {//必须全部消费同一个队列 - Collector.httpRequestPool.execute(new Consumer(Collector.collectorContainer, this)); - } - } - if ((!sync) && hasProducer) { - //有生产者才需要记录 - Collector.scheduled = Executors.newSingleThreadScheduledExecutor(); - Collector.scheduled - .scheduleAtFixedRate(new CollectorCounter(getEventSavePath()), 0, 2, TimeUnit.MINUTES); - if (EventConfig.saveFlag) { - Collector.scheduled.scheduleAtFixedRate( - new RangersFileCleaner(getEventFilePaths(), getEventSaveName(), getEventSaveMaxDays()), - 0, 12, TimeUnit.HOURS); - } - } - logger.info("Start DataRangers Cleaner/Record Thread"); - } - - - public static volatile Boolean IS_INIT = false; - - public void init() { - if (!IS_INIT) { - synchronized (DataRangersSDKConfigProperties.class) { - if (!IS_INIT) { - setLogger(); - setCommon(); - setHook(); - IS_INIT = true; - } - } - } - } - public CollectorQueue getUserQueue() { return userQueue; } @@ -504,17 +364,15 @@ public void setAppKeys(Map appKeys) { this.appKeys = appKeys; } - private void setHook() { - DataRangersSDKConfigProperties properties = this; - Runtime.getRuntime().addShutdownHook(new Thread( - () -> { - if(Collector.httpRequestPool != null){ - Collector.httpRequestPool.shutdown(); - } - - new Consumer(Collector.collectorContainer, properties).flush(); - })); + @Override + public String toString() { + return " domain:" + domain + " corePoolSize:" + corePoolSize + " httpTimout:" + httpTimeout + + " timeZone:" + timeZone + " timeOffset:" + timeOffset + " save:" + save + " queueSize:" + queueSize + + " send:" + send + " sendBatch:" + sendBatch + " batchSize:" + batchSize + " waitTimeMs:" + waitTimeMs + + " enable:" + enable + " eventSavePath:" + eventSavePath + " eventSaveName:" + eventSaveName + + " eventSaveMaxFileSize:" + eventSaveMaxFileSize + " eventSaveMaxDays:" + eventSaveMaxDays + + " userQueue:" + userQueue + " hasConsumer:" + hasConsumer + " hasProducer:" + hasProducer + + " env" + env + " sync:" + sync + " mode:" + mode + " kafka:" + kafka + " httpConfig" + httpConfig; } - } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java index d69348d..d8bf3d2 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java @@ -161,4 +161,13 @@ public Integer getKeepAliveTimeout() { public void setKeepAliveTimeout(Integer keepAliveTimeout) { this.keepAliveTimeout = keepAliveTimeout; } + + @Override + public String toString() { + return " requestTimeout:" + requestTimeout + "connectTimeout:" + connectTimeout + + " socketTimeout:" + socketTimeout + " keepAliveTimeout:" + keepAliveTimeout + + " maxTotal:" + maxTotal + " maxPerRoute" + maxPerRoute + + " keyMaterialPath" + keyMaterialPath + " keyPassword" + keyPassword + + " storePassword" + storePassword + " trustMaterialPath" + trustMaterialPath; + } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/KafkaConfig.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/KafkaConfig.java index 846e8a1..a1516cf 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/KafkaConfig.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/KafkaConfig.java @@ -34,4 +34,25 @@ public Map getProperties() { public void setProperties(Map properties) { this.properties = properties; } + + @Override + public String toString() { + StringBuilder kafkaSb = new StringBuilder(); + + kafkaSb.append(" topic:") + .append(topic) + .append(" bootstrapServers:") + .append(bootstrapServers); + + if (properties != null) { + kafkaSb.append("properties:"); + for (Map.Entry entry:properties.entrySet()) { + kafkaSb.append(entry.getKey()) + .append(":") + .append(entry.getValue()); + } + } + + return kafkaSb.toString(); + } } diff --git a/datarangers-sdk-core/src/main/java/test/AppEventCollectorTest.java b/datarangers-sdk-core/src/main/java/test/AppEventCollectorTest.java new file mode 100644 index 0000000..bc20200 --- /dev/null +++ b/datarangers-sdk-core/src/main/java/test/AppEventCollectorTest.java @@ -0,0 +1,38 @@ +package test; + +import com.datarangers.collector.AppEventCollector; +import com.datarangers.config.DataRangersSDKConfigProperties; +import com.datarangers.config.KafkaConfig; +import com.datarangers.config.SdkMode; +import org.junit.Test; + +/** + * @author qifeng.64343747@bytedance.com + * @date 2022-08-18 + */ +public class AppEventCollectorTest { + + @Test + public void initAppEventCollectorTest() { + //kafka + DataRangersSDKConfigProperties dataRangersSDKConfigProperties = new DataRangersSDKConfigProperties(); + dataRangersSDKConfigProperties.setMode(SdkMode.KAFKA); + KafkaConfig kafkaConfig = new KafkaConfig(); + kafkaConfig.setBootstrapServers("servers"); + dataRangersSDKConfigProperties.setKafka(kafkaConfig); + AppEventCollector appEventCollector = new AppEventCollector("app", dataRangersSDKConfigProperties, null); + + // http + dataRangersSDKConfigProperties.setMode(SdkMode.HTTP); + appEventCollector = new AppEventCollector("app", dataRangersSDKConfigProperties, null); + //file + dataRangersSDKConfigProperties.setMode(SdkMode.FILE); + appEventCollector = new AppEventCollector("app", dataRangersSDKConfigProperties, null); + + } + + @Test + public void sendEvent() { + + } +} From f4ea34fd81ad3eee507dd15d29f7b7bef6b9c954 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Wed, 28 Sep 2022 16:42:33 +0800 Subject: [PATCH 08/21] feature: change version to v1.5.6 snapshot --- datarangers-sdk-core/pom.xml | 4 ++-- datarangers-sdk-starter/pom.xml | 2 +- pom.xml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml index 00c4021..6a7d461 100644 --- a/datarangers-sdk-core/pom.xml +++ b/datarangers-sdk-core/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - 1.5.5-release + 1.5.6-SNAPSHOT 4.0.0 jar @@ -28,7 +28,7 @@ com.datarangers datarangers-sdk-core - 1.5.5-release + 1.5.6-SNAPSHOT diff --git a/datarangers-sdk-starter/pom.xml b/datarangers-sdk-starter/pom.xml index 2af176a..1bcf627 100644 --- a/datarangers-sdk-starter/pom.xml +++ b/datarangers-sdk-starter/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 1.5.5-release + 1.5.6-SNAPSHOT jar com.datarangers datarangers-sdk-starter diff --git a/pom.xml b/pom.xml index 6c43407..eee4483 100644 --- a/pom.xml +++ b/pom.xml @@ -4,13 +4,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 1.5.5-release + 1.5.6-SNAPSHOT com.datarangers datarangers-sdk pom - 1.5.5-release + 1.5.6-SNAPSHOT datarangers-sdk-core datarangers-sdk-starter From f3556d8483241e2b344a411fb811711a40d58394 Mon Sep 17 00:00:00 2001 From: qifeng Date: Wed, 28 Sep 2022 16:57:50 +0800 Subject: [PATCH 09/21] fix httpConfig toString --- .../src/main/java/com/datarangers/config/HttpConfig.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java index d8bf3d2..2f4bb13 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java @@ -166,8 +166,8 @@ public void setKeepAliveTimeout(Integer keepAliveTimeout) { public String toString() { return " requestTimeout:" + requestTimeout + "connectTimeout:" + connectTimeout + " socketTimeout:" + socketTimeout + " keepAliveTimeout:" + keepAliveTimeout + - " maxTotal:" + maxTotal + " maxPerRoute" + maxPerRoute + - " keyMaterialPath" + keyMaterialPath + " keyPassword" + keyPassword + - " storePassword" + storePassword + " trustMaterialPath" + trustMaterialPath; + " maxTotal:" + maxTotal + " maxPerRoute:" + maxPerRoute + + " keyMaterialPath:" + keyMaterialPath + " keyPassword:" + keyPassword + + " storePassword:" + storePassword + " trustMaterialPath:" + trustMaterialPath; } } From df5b0adf5209d6df9415fdb2ea71ce0ef484d9f1 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Thu, 29 Sep 2022 17:21:11 +0800 Subject: [PATCH 10/21] feature/add example --- datarangers-sdk-example/pom.xml | 58 ++++ .../example/AbstractSdkExample.java | 253 ++++++++++++++++++ .../example/BatchPriSdkExample.java | 79 ++++++ .../DataRangersSdkExampleApplication.java | 23 ++ .../example/FailCallbackExample.java | 50 ++++ .../example/FilePriSdkExample.java | 72 +++++ .../example/HttpPriSdkExample.java | 72 +++++ .../example/KafkaPriSdkExample.java | 81 ++++++ .../datarangers/example/SaasSdkExample.java | 82 ++++++ .../main/resources/application-pri.properties | 10 + .../resources/application-saas.properties | 19 ++ ...DataRangersSdkExampleApplicationTests.java | 23 ++ .../datarangers/example/SdkExampleTest.java | 111 ++++++++ 13 files changed, 933 insertions(+) create mode 100644 datarangers-sdk-example/pom.xml create mode 100644 datarangers-sdk-example/src/main/java/com/datarangers/example/AbstractSdkExample.java create mode 100644 datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java create mode 100644 datarangers-sdk-example/src/main/java/com/datarangers/example/DataRangersSdkExampleApplication.java create mode 100644 datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java create mode 100644 datarangers-sdk-example/src/main/java/com/datarangers/example/FilePriSdkExample.java create mode 100644 datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java create mode 100644 datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java create mode 100644 datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java create mode 100644 datarangers-sdk-example/src/main/resources/application-pri.properties create mode 100644 datarangers-sdk-example/src/main/resources/application-saas.properties create mode 100644 datarangers-sdk-example/src/test/java/com/datarangers/example/DataRangersSdkExampleApplicationTests.java create mode 100644 datarangers-sdk-example/src/test/java/com/datarangers/example/SdkExampleTest.java diff --git a/datarangers-sdk-example/pom.xml b/datarangers-sdk-example/pom.xml new file mode 100644 index 0000000..debd429 --- /dev/null +++ b/datarangers-sdk-example/pom.xml @@ -0,0 +1,58 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.7.4 + + + com.datarangers + datarangers-sdk-example + 0.0.1-SNAPSHOT + datarangers-sdk-example + datarangers-sdk-example + + 1.8 + + + + com.datarangers + datarangers-sdk-starter + 1.5.6-snapshot + + + org.springframework.boot + spring-boot-starter + + + + org.springframework.boot + spring-boot-starter-test + test + + + com.datarangers + datarangers-sdk-core + 1.5.6-SNAPSHOT + compile + + + junit + junit + 4.13.1 + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/AbstractSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/AbstractSdkExample.java new file mode 100644 index 0000000..00e2ef3 --- /dev/null +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/AbstractSdkExample.java @@ -0,0 +1,253 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import com.datarangers.collector.EventCollector; +import com.datarangers.event.*; + +import java.text.SimpleDateFormat; +import java.util.*; + +/** + * @Author zhangpeng.spin@bytedance.com + * @Date 2022/9/28 + */ +public abstract class AbstractSdkExample { + /** + * 获取 app应用上报的EventCollector + * + * @return EventCollector + */ + public abstract EventCollector getAppEventCollector(); + + /** + * 获取 web应用上报的EventCollector + * + * @return EventCollector + */ + public abstract EventCollector getWebEventCollector(); + + /** + * 获取 mp应用上报的EventCollector + * + * @return EventCollector + */ + public abstract EventCollector getMpEventCollector(); + + /** + * 上报事件 + * + * @param userUniqueId 用户ID + * @param appId 应用ID + */ + public void sendEvent(String userUniqueId, int appId) { + EventCollector appEventCollector = getAppEventCollector(); + appEventCollector + .sendEvent(userUniqueId, appId, null, "test_event_java_sdk", + new HashMap() {{ + put("date_time", new SimpleDateFormat("yyyyMMdd").format(new Date())); + put("current_time", + new SimpleDateFormat("yyyyMMdd HH:mm:ss").format(new Date())); + }}); + + System.out.println("end"); + } + + /** + * 传入自定义时间戳 + * + * @param userUniqueId + * @param appId + */ + public void sendEventWithLocalTimeMs(String userUniqueId, int appId, long localTimeMs) { + EventCollector appEventCollector = getAppEventCollector(); + appEventCollector + .sendEvent(userUniqueId, appId, null, "test_event_java_sdk", + new HashMap() {{ + put("date_time", new SimpleDateFormat("yyyyMMdd").format(new Date())); + put("current_time", + new SimpleDateFormat("yyyyMMdd HH:mm:ss").format(new Date())); + }}, + localTimeMs + ); + + System.out.println("end"); + } + + /** + * 上报事件，携带ab_sdk_version + * + * @param userUniqueId + * @param appId + */ + public void senEventWithAbSdk(String userUniqueId, int appId) { + EventCollector appEventCollector = getAppEventCollector(); + Map custom = new HashMap<>(); + Map eventParams = new HashMap<>(); + Header header = new HeaderV3.Builder().setCustom(custom).setAppId(appId) + .setUserUniqueId(userUniqueId).build(); + + for (int i = 0; i < 5; i++) { + Event event1 = new EventV3().setEvent("test_ab_sdk") + .setParams(eventParams).setUserId(userUniqueId) + .setLocalTimeMs(new Date().getTime()) + .setAbSdkVersion("12345"); + Event event2 = new EventV3().setEvent("test_ab_sdk") + .setParams(eventParams).setUserId(userUniqueId) + .setLocalTimeMs(new Date().getTime()) + .setAbSdkVersion("12345"); + appEventCollector.sendEvents(header, Arrays.asList(event1, event2)); + } + + } + + /** + * 上报用户属性 + * + * @param userUniqueId 用户id + * @param appId 应用id + */ + public void sendUserProfile(String userUniqueId, int appId) { + EventCollector appEventCollector = getAppEventCollector(); + appEventCollector.profileSet(userUniqueId, appId, new HashMap() {{ + put("profile_a", "param_11"); + put("profile_b", "param_22"); + put("profile_c", "param_33"); + put("profile_d", "param_44"); + }}); + } + + /** + * 上报item属性，需要先在系统创建item + * + * @param appId 应用ID + */ + public void sendItemProfile(int appId) { + EventCollector appEventCollector = getAppEventCollector(); + + List items = new ArrayList<>(); + items.add( + new BookItem("1000", "book") + .setName("Java") + .setPrice(100) + .setPublishDate("2010-10-11") + .setAuthors(Arrays.asList("zhangsan", "lisi")) + .setCategory("1")); + items.add( + new BookItem("1002", "book") + .setName("PHP") + .setPrice(100) + .setPublishDate("2021-07-20") + .setAuthors(Arrays.asList("zhangsan", "wanger")) + .setCategory("2")); + + appEventCollector.itemSet(appId, "book", items); + } + + /** + * 在事件中，携带item + * + * @param userUniqueId 用户id + * @param appId 应用id + */ + public void sendEventWithItem(String userUniqueId, int appId) { + EventCollector appEventCollector = getAppEventCollector(); + final List items = new ArrayList<>(); + items.add(new BookItem("1000", "book")); + items.add(new BookItem("1002", "book")); + appEventCollector + .sendEvent(userUniqueId, appId, null, "set_items", new HashMap() {{ + put("param1", "params"); + put("param2", items.get(0)); + put("param3", items.get(1)); + }}); + } + + /** + * item 对象 + */ + static class BookItem implements Item { + + private String itemId; + private String itemName; + + private String name; + private int price; + private String publishDate; + private List authors; + private String category; + + public BookItem() { + + } + + public BookItem(String itemId, String itemName) { + this.itemId = itemId; + this.itemName = itemName; + } + + @Override + public String getItemId() { + return itemId; + } + + @Override + public String getItemName() { + return itemName; + } + + public String getName() { + return name; + } + + public BookItem setName(String name) { + this.name = name; + return this; + } + + public int getPrice() { + return price; + } + + public BookItem setPrice(int price) { + this.price = price; + return this; + } + + + public List getAuthors() { + return authors; + } + + public BookItem setAuthors(List authors) { + this.authors = authors; + return this; + } + + public String getCategory() { + return category; + } + + public BookItem setCategory(String category) { + this.category = category; + return this; + } + + public String getPublishDate() { + return publishDate; + } + + public BookItem setPublishDate(String publishDate) { + this.publishDate = publishDate; + return this; + } + } +} diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java new file mode 100644 index 0000000..f511271 --- /dev/null +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import com.datarangers.collector.AppEventCollector; +import com.datarangers.collector.EventCollector; +import com.datarangers.config.DataRangersSDKConfigProperties; +import com.datarangers.config.SdkMode; + +/** + * @Author zhangpeng.spin@bytedance.com + * @Date 2022/9/28 + */ +public class BatchPriSdkExample extends AbstractSdkExample { + private EventCollector appEventCollector; + private EventCollector webEventCollector; + private EventCollector mpEventCollector; + + BatchPriSdkExample() { + DataRangersSDKConfigProperties properties = new DataRangersSDKConfigProperties(); + // 设置模式 + properties.setMode(SdkMode.HTTP); + + // 设置domain和host + properties.setDomain(System.getenv("SDK_DOMAIN")); + properties.getHeaders().put("HOST", System.getenv("SDK_HOST")); + + // 设置batch + properties.setSendBatch(true); + + // 可以根据需要进行调试batchSize,一般使用默认的即可 + // properties.setBatchSize(20); + // properties.setWaitTimeMs(100); + + // 初始化collector + appEventCollector = new AppEventCollector("app", properties); + webEventCollector = new AppEventCollector("web", properties); + mpEventCollector = new AppEventCollector("mp", properties); + } + + + @Override + public EventCollector getAppEventCollector() { + return appEventCollector; + } + + @Override + public EventCollector getWebEventCollector() { + return webEventCollector; + } + + @Override + public EventCollector getMpEventCollector() { + return mpEventCollector; + } + + public static void main(String[] args) { + BatchPriSdkExample sdkExample = new BatchPriSdkExample(); + String userUniqueId = "test_sdk_user1"; + int appId = 10000000; + + // 发送事件，时间发生时间为send方法调用的时间 + sdkExample.sendEvent(userUniqueId, appId); + sdkExample.senEventWithAbSdk(userUniqueId, appId); + sdkExample.sendUserProfile(userUniqueId, appId); + + // 指定localTimeMs时间,即事件发生时间 + long localTimeMs = System.currentTimeMillis(); + sdkExample.sendEventWithLocalTimeMs(userUniqueId, appId, localTimeMs); + } +} diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/DataRangersSdkExampleApplication.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/DataRangersSdkExampleApplication.java new file mode 100644 index 0000000..a820848 --- /dev/null +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/DataRangersSdkExampleApplication.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class DataRangersSdkExampleApplication { + + public static void main(String[] args) { + SpringApplication.run(DataRangersSdkExampleApplication.class, args); + } + +} diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java new file mode 100644 index 0000000..2eb1131 --- /dev/null +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import com.datarangers.collector.AppEventCollector; +import com.datarangers.collector.EventCollector; +import com.datarangers.config.DataRangersSDKConfigProperties; +import com.datarangers.config.SdkMode; +import com.datarangers.sender.Callback; + +/** + * @Author zhangpeng.spin@bytedance.com + * @Date 2022/9/29 + */ +public abstract class FailCallbackExample extends AbstractSdkExample { + private EventCollector appEventCollector; + + FailCallbackExample() { + DataRangersSDKConfigProperties properties = new DataRangersSDKConfigProperties(); + // 设置模式 + properties.setMode(SdkMode.HTTP); + + // 设置domain和host + properties.setDomain(System.getenv("SDK_DOMAIN")); + properties.getHeaders().put("HOST", System.getenv("SDK_HOST")); + + // 自定义失败处理方式 + Callback callback = failedData -> { + System.out.println("message: " + failedData.getMessage()); + System.out.println("cause: " + failedData.getCause()); + System.out.println("exception: " + failedData.getException()); + }; + + // 初始化collector + appEventCollector = new AppEventCollector("app", properties, callback); + } + + @Override + public EventCollector getAppEventCollector() { + return appEventCollector; + } +} diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/FilePriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/FilePriSdkExample.java new file mode 100644 index 0000000..63d3b61 --- /dev/null +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/FilePriSdkExample.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import com.datarangers.collector.AppEventCollector; +import com.datarangers.collector.EventCollector; +import com.datarangers.config.DataRangersSDKConfigProperties; +import com.datarangers.config.SdkMode; + +/** + * @Author zhangpeng.spin@bytedance.com + * @Date 2022/9/28 + */ +public class FilePriSdkExample extends AbstractSdkExample { + private EventCollector appEventCollector; + private EventCollector webEventCollector; + private EventCollector mpEventCollector; + + FilePriSdkExample() { + DataRangersSDKConfigProperties properties = new DataRangersSDKConfigProperties(); + // 设置模式 + properties.setMode(SdkMode.HTTP); + + // 设置domain和host + properties.setDomain(System.getenv("SDK_DOMAIN")); + properties.getHeaders().put("HOST", System.getenv("SDK_HOST")); + + // 初始化collector + appEventCollector = new AppEventCollector("app", properties); + webEventCollector = new AppEventCollector("web", properties); + mpEventCollector = new AppEventCollector("mp", properties); + } + + + @Override + public EventCollector getAppEventCollector() { + return appEventCollector; + } + + @Override + public EventCollector getWebEventCollector() { + return webEventCollector; + } + + @Override + public EventCollector getMpEventCollector() { + return mpEventCollector; + } + + public static void main(String[] args) { + FilePriSdkExample sdkExample = new FilePriSdkExample(); + String userUniqueId = "test_sdk_user1"; + int appId = 10000000; + + // 发送事件，时间发生时间为send方法调用的时间 + sdkExample.sendEvent(userUniqueId, appId); + sdkExample.senEventWithAbSdk(userUniqueId, appId); + sdkExample.sendUserProfile(userUniqueId, appId); + + // 指定localTimeMs时间,即事件发生时间 + long localTimeMs = System.currentTimeMillis(); + sdkExample.sendEventWithLocalTimeMs(userUniqueId, appId, localTimeMs); + } +} diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java new file mode 100644 index 0000000..08fcafc --- /dev/null +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import com.datarangers.collector.AppEventCollector; +import com.datarangers.collector.EventCollector; +import com.datarangers.config.DataRangersSDKConfigProperties; +import com.datarangers.config.SdkMode; + +/** + * @Author zhangpeng.spin@bytedance.com + * @Date 2022/9/28 + */ +public class HttpPriSdkExample extends AbstractSdkExample { + private EventCollector appEventCollector; + private EventCollector webEventCollector; + private EventCollector mpEventCollector; + + HttpPriSdkExample() { + DataRangersSDKConfigProperties properties = new DataRangersSDKConfigProperties(); + // 设置模式 + properties.setMode(SdkMode.HTTP); + + // 设置domain和host + properties.setDomain(System.getenv("SDK_DOMAIN")); + properties.getHeaders().put("HOST", System.getenv("SDK_HOST")); + + // 初始化collector + appEventCollector = new AppEventCollector("app", properties); + webEventCollector = new AppEventCollector("web", properties); + mpEventCollector = new AppEventCollector("mp", properties); + } + + + @Override + public EventCollector getAppEventCollector() { + return appEventCollector; + } + + @Override + public EventCollector getWebEventCollector() { + return webEventCollector; + } + + @Override + public EventCollector getMpEventCollector() { + return mpEventCollector; + } + + public static void main(String[] args) { + HttpPriSdkExample sdkExample = new HttpPriSdkExample(); + String userUniqueId = "test_sdk_user1"; + int appId = 10000000; + + // 发送事件，时间发生时间为send方法调用的时间 + sdkExample.sendEvent(userUniqueId, appId); + sdkExample.senEventWithAbSdk(userUniqueId, appId); + sdkExample.sendUserProfile(userUniqueId, appId); + + // 指定localTimeMs时间,即事件发生时间 + long localTimeMs = System.currentTimeMillis(); + sdkExample.sendEventWithLocalTimeMs(userUniqueId, appId, localTimeMs); + } +} diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java new file mode 100644 index 0000000..fdbf3fe --- /dev/null +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import com.datarangers.collector.AppEventCollector; +import com.datarangers.collector.EventCollector; +import com.datarangers.config.DataRangersSDKConfigProperties; +import com.datarangers.config.KafkaConfig; +import com.datarangers.config.SdkMode; + +/** + * @Author zhangpeng.spin@bytedance.com + * @Date 2022/9/28 + */ +public class KafkaPriSdkExample extends AbstractSdkExample { + private EventCollector appEventCollector; + private EventCollector webEventCollector; + private EventCollector mpEventCollector; + + KafkaPriSdkExample() { + DataRangersSDKConfigProperties properties = new DataRangersSDKConfigProperties(); + + // 设置模式 + properties.setMode(SdkMode.KAFKA); + + // 设置kafka + KafkaConfig kafkaConfig = new KafkaConfig(); + properties.setKafka(kafkaConfig); + + // 设置 BootstrapServers, etc: 127.0.0.1:9192,localhost:9192 + kafkaConfig.setBootstrapServers(System.getenv("SDK_KAFKA_BOOTSTRAP_SERVERS")); + + // 如果需要设置其他属性，可以进行设置 + // Map kafkaProperties = new HashMap<>(); + // kafkaConfig.setProperties(kafkaProperties); + + // 初始化collector + appEventCollector = new AppEventCollector("app", properties); + webEventCollector = new AppEventCollector("web", properties); + mpEventCollector = new AppEventCollector("mp", properties); + } + + + @Override + public EventCollector getAppEventCollector() { + return appEventCollector; + } + + @Override + public EventCollector getWebEventCollector() { + return webEventCollector; + } + + @Override + public EventCollector getMpEventCollector() { + return mpEventCollector; + } + + public static void main(String[] args) { + KafkaPriSdkExample sdkExample = new KafkaPriSdkExample(); + String userUniqueId = "test_sdk_user1"; + int appId = 10000000; + + // 发送事件，时间发生时间为send方法调用的时间 + sdkExample.sendEvent(userUniqueId, appId); + sdkExample.senEventWithAbSdk(userUniqueId, appId); + sdkExample.sendUserProfile(userUniqueId, appId); + + // 指定localTimeMs时间,即事件发生时间 + long localTimeMs = System.currentTimeMillis(); + sdkExample.sendEventWithLocalTimeMs(userUniqueId, appId, localTimeMs); + } +} diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java new file mode 100644 index 0000000..485ba81 --- /dev/null +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import com.datarangers.collector.AppEventCollector; +import com.datarangers.collector.EventCollector; +import com.datarangers.config.DataRangersSDKConfigProperties; +import com.datarangers.config.SdkMode; + +/** + * @Author zhangpeng.spin@bytedance.com + * @Date 2022/9/28 + */ +public class SaasSdkExample extends AbstractSdkExample { + private EventCollector appEventCollector; + private EventCollector webEventCollector; + private EventCollector mpEventCollector; + + SaasSdkExample() { + DataRangersSDKConfigProperties properties = new DataRangersSDKConfigProperties(); + // 设置模式 + properties.setMode(SdkMode.HTTP); + + // 设置domain和appKey, 不需要设置HOST + properties.setDomain("https://mcs.ctobsnssdk.com"); + + // 可以设置多个app + properties.getAppKeys().put(Integer.valueOf(System.getenv("SDK_APP_1")), System.getenv("SDK_APP_KEY_1")); + + // 设置openapi domain, AK,SK + properties.getOpenapiConfig().setDomain("https://analytics.volcengineapi.com"); + properties.getOpenapiConfig().setAk(System.getenv("OPENAPI_AK")); + properties.getOpenapiConfig().setSk(System.getenv("OPENAPI_SK")); + + + // 初始化collector + appEventCollector = new AppEventCollector("app", properties); + webEventCollector = new AppEventCollector("web", properties); + mpEventCollector = new AppEventCollector("mp", properties); + } + + + @Override + public EventCollector getAppEventCollector() { + return appEventCollector; + } + + @Override + public EventCollector getWebEventCollector() { + return webEventCollector; + } + + @Override + public EventCollector getMpEventCollector() { + return mpEventCollector; + } + + public static void main(String[] args) { + SaasSdkExample sdkExample = new SaasSdkExample(); + String userUniqueId = "test_sdk_user1"; + int appId = Integer.valueOf(System.getenv("SDK_APP_1")); + + // 发送事件，时间发生时间为send方法调用的时间 + sdkExample.sendEvent(userUniqueId, appId); + sdkExample.senEventWithAbSdk(userUniqueId, appId); + + // 需要保证先在系统新增用户属性 + sdkExample.sendUserProfile(userUniqueId, appId); + + // 指定localTimeMs时间,即事件发生时间 + long localTimeMs = System.currentTimeMillis(); + sdkExample.sendEventWithLocalTimeMs(userUniqueId, appId, localTimeMs); + } +} diff --git a/datarangers-sdk-example/src/main/resources/application-pri.properties b/datarangers-sdk-example/src/main/resources/application-pri.properties new file mode 100644 index 0000000..b493156 --- /dev/null +++ b/datarangers-sdk-example/src/main/resources/application-pri.properties @@ -0,0 +1,10 @@ +## Http\u8BF7\u6C42Header\u5185\u5BB9,Host\u4E3A\u5FC5\u987B +datarangers.sdk.mode=http +datarangers.sdk.headers.Host=${SDK_HOST} + +# batch +#datarangers.sdk.sendBatch=true +#datarangers.sdk.batchSize=16 +#datarangers.sdk.waitTimeMs=100 + + diff --git a/datarangers-sdk-example/src/main/resources/application-saas.properties b/datarangers-sdk-example/src/main/resources/application-saas.properties new file mode 100644 index 0000000..c9f7af0 --- /dev/null +++ b/datarangers-sdk-example/src/main/resources/application-saas.properties @@ -0,0 +1,19 @@ +datarangers.sdk.env=saas +datarangers.sdk.mode=http + +# \u670D\u52A1\u5668ip\u6216\u57DF\u540D +datarangers.sdk.domain=https://mcs.ctobsnssdk.com +# app key +datarangers.sdk.appKeys.${SDK_APP_1}=${SDK_APP_KEY_1} + +# openapi\u7684domain +datarangers.sdk.openapiConfig.domain=https://analytics.volcengineapi.com + +# openapi\u7684ak, sk +datarangers.sdk.openapiConfig.ak=${OPENAPI_AK} +datarangers.sdk.openapiConfig.sk=${OPENAPI_SK} + + + + + diff --git a/datarangers-sdk-example/src/test/java/com/datarangers/example/DataRangersSdkExampleApplicationTests.java b/datarangers-sdk-example/src/test/java/com/datarangers/example/DataRangersSdkExampleApplicationTests.java new file mode 100644 index 0000000..200e3b6 --- /dev/null +++ b/datarangers-sdk-example/src/test/java/com/datarangers/example/DataRangersSdkExampleApplicationTests.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class DataRangersSdkExampleApplicationTests { + + @Test + void contextLoads() { + } + +} diff --git a/datarangers-sdk-example/src/test/java/com/datarangers/example/SdkExampleTest.java b/datarangers-sdk-example/src/test/java/com/datarangers/example/SdkExampleTest.java new file mode 100644 index 0000000..b381885 --- /dev/null +++ b/datarangers-sdk-example/src/test/java/com/datarangers/example/SdkExampleTest.java @@ -0,0 +1,111 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import com.datarangers.collector.AppEventCollector; +import com.datarangers.collector.EventCollector; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +@ActiveProfiles("${PROFILE}") +class SdkExampleTest extends AbstractSdkExample { + + @Autowired + @Qualifier("appEventCollector") + private AppEventCollector appEventCollector; + + @Autowired + @Qualifier("webEventCollector") + private AppEventCollector webEventCollector; + + @Autowired + @Qualifier("mpEventCollector") + private AppEventCollector mpEventCollector; + + private int appId; + private String userUniqueId; + + @BeforeEach + public void beforeEach() { + appId = 10000000; + userUniqueId = "test_sdk_user2"; + } + + @AfterEach + public void afterEach() { + // 优雅退出 + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + @AfterAll + public static void clearUp() { + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + @Test + public void sendEvent() { + sendEvent(userUniqueId, appId); + } + + @Test + public void senEventWithAbSdk() { + senEventWithAbSdk(userUniqueId, appId); + } + + @Test + public void sendUserProfile() { + sendUserProfile(userUniqueId, appId); + } + + @Test + public void sendItemProfile() { + sendEventWithItem(userUniqueId, appId); + } + + @Test + public void sendEventWithItem() { + sendEventWithItem(userUniqueId, appId); + } + + @Override + public EventCollector getAppEventCollector() { + return appEventCollector; + } + + @Override + public EventCollector getWebEventCollector() { + return webEventCollector; + } + + @Override + public EventCollector getMpEventCollector() { + return mpEventCollector; + } + +} From 213b33e5ec26299e91b31d7134655daedf7addc0 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Thu, 29 Sep 2022 18:30:59 +0800 Subject: [PATCH 11/21] feature: opt readme --- .gitignore | 1 + README.md | 332 +---------------------------------------------------- 2 files changed, 6 insertions(+), 327 deletions(-) diff --git a/.gitignore b/.gitignore index 4bfa550..dd18300 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea/ *.iml target/ +logs/ diff --git a/README.md b/README.md index fad4f13..caad7ea 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # DataRangers ## 项目背景 -datarangers-sdk-java是 [DataFinder](https://www.volcengine.com/product/datafinder) 的用户行为采集服务端SDK。 +datarangers-sdk-java 是 [DataFinder](https://www.volcengine.com/product/datafinder) 的用户行为采集服务端SDK。 服务端埋点支持在客户的服务端进行埋点采集和上报，作为客户端埋点的补充或替代，其支持的典型场景包括： 1. 客户端埋点+服务端埋点组合：该场景下，服务端埋点一般用来弥补客户端埋点覆盖不到的部分数据，是目前最常见的使用场景。 @@ -17,8 +17,6 @@ datarangers-sdk-java是 [DataFinder](https://www.volcengine.com/product/datafind ``` -version是sdk的版本号，当前最新的版本为1.5.5-release。 - 火山引擎仓库地址： ```xml @@ -30,332 +28,12 @@ version是sdk的版本号，当前最新的版本为1.5.5-release。 ``` -### 2. 配置SDK -DataRangers SDK需要进行一定的参数配置才能够使用，具体需要配置的参数为： -* domain：datarangers的域名或者ip，支持http和https，例如为 https://www.xxx.com，在私有化环境中，需要修改为对应的sdk上报域名或者使用DataRangers服务器的ip地址。在saas环境中需要修改成对应的域名： - * 中国区：https://mcs.ctobsnssdk.com - * sg(新加坡): https://mcs.tobsnssdk.com - * va(美东): https://mcs.itobsnssdk.com -* save：bool型变量，表示是否保存到文件： - * true：保存到文件，但是需要配置LogAgent完成事件上报功能,需要额外定义： - * eventSaveName：保存日志的文件名，需要保证文件的写权限。 - * eventSavePath：保存日志的文件路径，需要保证写权限和创建文件的权限。 - * eventSaveMaxFileSize：表示需要保存的日志文件的最大文件大小，单位为MB。 - * eventFilePaths：表示需要保存的日志文件的位置，为一个字符串数组，数组中的每一个值都表示一个路径，用户将日志文件写到不同的文件夹下，可以配合多个LogAgent实例使用。注意：如果定义了该数组，则eventSavePath不会生效。 - * false：使用http模式进行异步上报： - * httpTimeout：Http的超时时间，单位为ms。 - * headers：Map类型，Http请求的Header中的字段，必填为Host ，Host在DataRangers安装中进行定义 -* mode: 枚举值，支持kafka,http,file。建议使用新的该配置。当mode和save同时存在的时候，以mode为准 - * http 等同于save=false - * file 等同于save=true - * kafka，支持直接通过kafka进行上报，当使用该模式的时候，需要配置kafka的上报地址： - * bootstrapServers： kafka的地址 - * properties: 是一个map，需要配置的其他的kafka properties。kafkaProducer的参数参考：https://kafka.apache.org/0102/documentation.html#producerconfigs - -如果您使用了Spring框架，则可以参考的配置如下： -```xml - - - - - - - - - - - - - - - - - - - - - - - - - -``` - -如果您使用SpringBoot框架，我们提供了一个封装完成的的starter包，您可以在pom中通过如下方式引入： -```xml - - com.datarangers - datarangers-sdk-starter - 1.5.5-release - -``` - -并在properties文件中对sdk进行配置 -```properties -# 使能sdk功能,为false就disable sdk功能 -datarangers.sdk.enable=true - -# privatization 表示是私有化环境， saas表示是saas环境，默认是私有化环境。sdk会根据配置的datarangers.sdk.domain自动识别是否是saas环境，该配置可选 -# datarangers.sdk.env=privatization -# rangers的ip或域名 -datarangers.sdk.domain=http://domain - -# datarangers.sdk.headers为http请求中headers字段内容,在私有化环境中必须要添加Host，而在saas环境中 不能配置Host,其他如果需要设置的可以选填 -# Host的配置在安装部署的那台机器上，查看/home/datarangers/DataRangersDeploy/conf_rangers.yml中配置项sdk.report.host -datarangers.sdk.headers.Host=host - -# 如果在saas环境中，需要配置appkey -# datarangers.sdk.appKeys.${appId}=xxx - -# 如果是在saas环境中，需要配置openapi, 私有化环境中可以不配置 -# openapi的domain， 国内: https://analytics.volcengineapi.com，国际是: https://analytics.byteplusapi.com -# datarangers.sdk.openapiConfig.domain=xxx - -# openapi的ak, sk -# datarangers.sdk.openapiConfig.ak=xxx -# datarangers.sdk.openapiConfig.sk=xxx - -# 是否保存到本地,如果需要配合logagent使用需要将其定义为true -datarangers.sdk.save=true -# 异步方式的发送线程数量,如果为logagent模式请设置为1 -datarangers.sdk.threadCount=4 -# 异步方式的发送核心线程数量，建议corePoolSize 跟threadCount 配置成一样 -datarangers.sdk.corePoolSize=4 -# 异步方式队列长度 -datarangers.sdk.queueSize=102400 - -# 是否使用批量发送,默认为false -#datarangers.sdk.sendBatch=true - -# 批量发送的大小 -#datarangers.sdk.batchSize=16 - -# 批量的等待时间，当批量达到batchSize，或者等待时间超过waitTimeMs，就立刻发送 -#datarangers.sdk.waitTimeMs=100 - -# 保存日志文件路径 -datarangers.sdk.eventSavePath=logs/ -# 保存日志文件名 -datarangers.sdk.eventSaveName=datarangers -# 最多保存的单个日志的大小,单位MB -datarangers.sdk.eventSaveMaxFileSize=256 - -# client是否需要进行ssl证书认证，默认为false，false表示需要进行证书认证，这也是jdk自身的默认标准行为。如果访问https, 需要把证书导入到证书库里面，默认使用的是jdk的证书库，建议客户使用这种方式；如果不想导入的话，可以设置trustDisable为true，sdk会通过设置一个自定义的trustManager跳过认证 -# datarangers.sdk.httpConfig.trustDisable=false - -# 自定义证书路径和密码，false表示使用jdk自身的默认路径 -# datarangers.sdk.httpConfig.customKeyTrustEnable=false - -# 配置证书 -# datarangers.sdk.httpConfig.keyMaterialPath=xxx -# datarangers.sdk.httpConfig.keyPassword=xxx -# datarangers.sdk.httpConfig.storePassword=xxx -# datarangers.sdk.httpConfig.trustMaterialPath=xxx - -# self for selfTrustStrategy, default is all -# datarangers.sdk.httpConfig.trustStrategy=xxx - -# http 超时配置 -# http request timeout， 单位是毫秒 -# datarangers.sdk.httpConfig.requestTimeout=10000 -# http connect timeout， 单位是毫秒 -# datarangers.sdk.httpConfig.connectTimeout=10000 -# http socket timeout， 单位是毫秒 -# datarangers.sdk.httpConfig.socketTimeout=20000 -# http keep alive time, 单位是秒 -# datarangers.sdk.httpConfig.keepAliveTimeout=180 - -# http 连接配置 -# 连接池最大连接数 -# datarangers.sdk.httpConfig.maxTotal=1000 -# 每一个 host 的最大连接数 -# datarangers.sdk.httpConfig.maxPerRoute=100 - -# kafka 配置 -# 设置模式为kafka -# datarangers.sdk.mode=kafka - -# 配置发送的kafka topic，没有配置时，使用默认sdk_origin_event， -# datarangers.sdk.kafka.topic=sdk_origin_event - -# 配置发送的地址,ip需要替换成真实的ip -# datarangers.sdk.kafka.bootstrapServers={ip1}:9192,{ip2}:9192 - -# 如果有需要，配置其他的属性, 形式为：datarangers.sdk.kafka.properties.${key}=${value}, 比如配置重试次数。 -# kafkaProducer的参数参考：https://kafka.apache.org/0102/documentation.html#producerconfigs -# 重试次数 -# datarangers.sdk.kafka.properties.retries=3 -``` - -### 3. 使用SDK -使用时需要先注入Bean，Bean有三种类型，如下： -```java -// App -@Resource(name = "appEventCollector") -private EventCollector appEventCollector; -// Web -@Resource(name = "webEventCollector") -private EventCollector webEventCollector; -// 小程序 -@Resource(name = "mpEventCollector") -private EventCollector mpEventCollector; -``` - -如果您已经注入完成了，则可以调用bean进行事件发送。发送的接口为： -```java -/** - * 功能描述: 异步发送事件 - * - * @param appId 应用id - * @param custom 用户自定义公共参数 - * @param eventName 事件名称 - * @param eventParams 事件参数 - * @param userUniqueId 用户uuid - * @return: void - * @date: 2020/8/26 12:24 - */ -void sendEvent(String userUniqueId, int appId, Map custom, String eventName, Map eventParams); - -/** - * 功能描述: 批量发送事件 - * - * @param header 事件的公共属性，可以通过调用HeaderV3.Builder().build()构建一个header - * @param events 事件数组 一般不推荐自己构建事件数组，我们推荐使用EventsBuilder这个类对多事件进行构造，并调用build方法生成事件数组 - * @return: void - * @date: 2020/12/25 15:57 - */ -void sendEvents(Header header, List events); - -/** - * 功能描述: 发送单条事件 - * - * @param header 事件的公共属性，可以通过调用HeaderV3.Builder().build()构建一个header - * @param eventName 事件名 - * @param eventParams 事件参数 - * @return: void - * @date: 2020/9/28 22:00 - */ -void sendEvent(Header header, String eventName, Map eventParams); - -/** - * 功能描述: 批量发送事件 - * - * @param header 事件的公共属性，可以通过调用HeaderV3.Builder().build()构建一个header - * @param eventName 事件名数组，需要与eventParams数组长度相同 - * @return: void - * @date: 2020/12/25 15:59 - */ -void sendEvent(Header header, List eventName, List> eventParams); - -/** - * 功能描述: 对userUniqueId的用户进行profile属性设置 - * - * @param appId app id - * @param userUniqueId 用户id - * @param eventParams 需要设置的用户属性 - * @return: void - * @date: 2020/12/23 10:43 - */ -void profileSet(String userUniqueId, int appId, Map eventParams); - -void profileSetOnce(String userUniqueId, int appId, Map eventParams); - -void profileIncrement(String userUniqueId, int appId, Map eventParams); - -void profileAppend(String userUniqueId, int appId, Map eventParams); - -/** - * 功能描述: 删除用户的属性 - * @param appId app id - * @param userUniqueId uuid - * @param params 需要删除的用户属性名 - * @return: void - * @date: 2020/12/25 16:11 - */ -void profileUnset(String userUniqueId, int appId, List params); +### 2. 使用方式 +参考官方文档 [DataFinder](https://www.volcengine.com/docs/6285/75430) -/** - * 功能描述: 对业务对象进行设置 - * - * @param appId app id - * @param name 业务对象的名称 - * @param items 业务对象的类，需要继承Items类,注意 - * @return: void - * @date: 2020/12/23 10:47 - */ -void itemSet(int appId, String name, List items); +### 3. Demo +参考 [datarangers-sdk-example](https://github.com/volcengine/datarangers-sdk-java/tree/main/datarangers-sdk-example) 代码样例 -/** - * 功能描述: 删除item的属性 - * @param appId - * @param id - * @param name - * @param params 需要删除的item属性 - * @return: void - * @date: 2020/12/25 16:13 - */ -void itemUnset(int appId, String id, String name, List params); -``` - -## 使用示例 -1. 发送普通事件 -```java -@Resource(name = "appEventCollector") -private EventCollector appEventCollector; - -appEventCollector.sendEvent("uuid2", 10000000, null, "test_event_java_sdk", new HashMap() {{ - put("date_time", LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"))); - put("current_time", LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd HH:mm:ss"))); -}}); - -``` - -2. 设置用户属性 -```java -eventCollector.profileSet("uuid-1", 10000028, new HashMap() {{ - put("profile_1", "param_1"); - put("profile_2", "param_2"); - put("profile_3", "param_3"); - put("profile_4", "param_4"); -}}); -``` - -3. 设置Item属性 -```java -List items = new ArrayList<>(); -items.add(new BookItems("1000", "book").setName("Java").setPrice(100).setPublishDate(LocalDate.now()).setAuthors(author).setCategory("computer")); -items.add(new BookItems("1002", "book").setName("PHP").setPrice(100).setPublishDate(LocalDate.now()).setAuthors(author).setCategory("computer")); -eventCollector.itemSet(10000028, "book", items); -``` - -4. 发送携带item的事件 -```java -List items = new ArrayList<>(); -items.add(new BookItems("1000", "book")); -items.add(new BookItems("1002", "book")); -items.add(new PhoneItems("1002", "phone")); -eventCollector.sendEvent("user-001", 10000028, null, "set_items", new HashMap() {{ - put("param1", "params"); - put("param2", items.get(0)); - put("param3", items.get(1)); - put("param4", items.get(2)); -}}); -``` - -5. 使用header上报事件 -```java -// 可以设置userUniqueId和deviceId等，具体字段可以查看Header类 -Map custom = new HashMap(); -Map eventParams = new HashMap(); -Header header = new HeaderV3.Builder() - .setCustom(custom) - .setAppId(10000000) - .setUserUniqueId("uuid-1") - .setDeviceId(1231232131313123L) - .build(); - -appEventCollector.sendEvent(header, "test_event_java_sdk_header", eventParams); -``` ## 注意事项 * 当前sdk版本没有主动清理日志的功能，需要手动清理日志 From 59bab3d1e49b48e53b0bb6213a55f603d435c0c3 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Fri, 30 Sep 2022 15:38:52 +0800 Subject: [PATCH 12/21] feature: opt example --- README.md | 1 + .../example/BatchPriSdkExample.java | 2 + .../example/FailCallbackExample.java | 2 + .../example/FilePriSdkExample.java | 11 ++--- .../example/HttpPriSdkExample.java | 5 ++- .../example/KafkaPriSdkExample.java | 5 ++- .../datarangers/example/SaasSdkExample.java | 5 ++- .../main/resources/application-pri.properties | 10 ----- .../resources/application-prifile.properties | 11 +++++ .../resources/application-prihttp.properties | 30 ++++++++++++++ .../application-prihttpbatch.properties | 30 ++++++++++++++ .../resources/application-prikafka.properties | 7 ++++ .../src/main/resources/application.properties | 11 +++++ .../src/main/resources/application.properties | 41 +++++++++++++++++++ 14 files changed, 152 insertions(+), 19 deletions(-) delete mode 100644 datarangers-sdk-example/src/main/resources/application-pri.properties create mode 100644 datarangers-sdk-example/src/main/resources/application-prifile.properties create mode 100644 datarangers-sdk-example/src/main/resources/application-prihttp.properties create mode 100644 datarangers-sdk-example/src/main/resources/application-prihttpbatch.properties create mode 100644 datarangers-sdk-example/src/main/resources/application-prikafka.properties create mode 100644 datarangers-sdk-example/src/main/resources/application.properties create mode 100644 datarangers-sdk-starter/src/main/resources/application.properties diff --git a/README.md b/README.md index caad7ea..0efc3bf 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ datarangers-sdk-java 是 [DataFinder](https://www.volcengine.com/product/datafin ## 注意事项 * 当前sdk版本没有主动清理日志的功能，需要手动清理日志 +* 上报事件，需要注意下事件发生时间 ## License diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java index f511271..0465aa9 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java @@ -16,6 +16,8 @@ import com.datarangers.config.SdkMode; /** + * HTTP模式的特殊场景，当跨网络传输，http 时延比较大的场景，可以使用批量上报的方式来提高性能 + * * @Author zhangpeng.spin@bytedance.com * @Date 2022/9/28 */ diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java index 2eb1131..403c04e 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java @@ -17,6 +17,8 @@ import com.datarangers.sender.Callback; /** + * 适用于需要自定义发送失败后处理的场景 + * * @Author zhangpeng.spin@bytedance.com * @Date 2022/9/29 */ diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/FilePriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/FilePriSdkExample.java index 63d3b61..6a98a34 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/FilePriSdkExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/FilePriSdkExample.java @@ -16,6 +16,7 @@ import com.datarangers.config.SdkMode; /** + * 使用 File 文件模式，需要配合 logagent 一起使用 * @Author zhangpeng.spin@bytedance.com * @Date 2022/9/28 */ @@ -27,11 +28,10 @@ public class FilePriSdkExample extends AbstractSdkExample { FilePriSdkExample() { DataRangersSDKConfigProperties properties = new DataRangersSDKConfigProperties(); // 设置模式 - properties.setMode(SdkMode.HTTP); + properties.setMode(SdkMode.FILE); - // 设置domain和host - properties.setDomain(System.getenv("SDK_DOMAIN")); - properties.getHeaders().put("HOST", System.getenv("SDK_HOST")); + // 设置路径 + properties.setEventSavePath("logs/"); // 初始化collector appEventCollector = new AppEventCollector("app", properties); @@ -62,7 +62,8 @@ public static void main(String[] args) { // 发送事件，时间发生时间为send方法调用的时间 sdkExample.sendEvent(userUniqueId, appId); - sdkExample.senEventWithAbSdk(userUniqueId, appId); + + // 上报用户属性 sdkExample.sendUserProfile(userUniqueId, appId); // 指定localTimeMs时间,即事件发生时间 diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java index 08fcafc..3c851d5 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java @@ -16,6 +16,8 @@ import com.datarangers.config.SdkMode; /** + * 在私有化场景，直接使用 Http 模式使用进行发送事件 + * * @Author zhangpeng.spin@bytedance.com * @Date 2022/9/28 */ @@ -62,7 +64,8 @@ public static void main(String[] args) { // 发送事件，时间发生时间为send方法调用的时间 sdkExample.sendEvent(userUniqueId, appId); - sdkExample.senEventWithAbSdk(userUniqueId, appId); + + // 上报用户属性 sdkExample.sendUserProfile(userUniqueId, appId); // 指定localTimeMs时间,即事件发生时间 diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java index fdbf3fe..0475b96 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java @@ -17,6 +17,8 @@ import com.datarangers.config.SdkMode; /** + * 在私有化场景，直接使用 Kafka 模式使用进行发送事件，私有化在同一个内网中，对qps有极高的要求 + * * @Author zhangpeng.spin@bytedance.com * @Date 2022/9/28 */ @@ -71,7 +73,8 @@ public static void main(String[] args) { // 发送事件，时间发生时间为send方法调用的时间 sdkExample.sendEvent(userUniqueId, appId); - sdkExample.senEventWithAbSdk(userUniqueId, appId); + + // 上报用户属性 sdkExample.sendUserProfile(userUniqueId, appId); // 指定localTimeMs时间,即事件发生时间 diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java index 485ba81..585b23e 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java @@ -16,6 +16,8 @@ import com.datarangers.config.SdkMode; /** + * saas 使用场景 + * * @Author zhangpeng.spin@bytedance.com * @Date 2022/9/28 */ @@ -70,9 +72,8 @@ public static void main(String[] args) { // 发送事件，时间发生时间为send方法调用的时间 sdkExample.sendEvent(userUniqueId, appId); - sdkExample.senEventWithAbSdk(userUniqueId, appId); - // 需要保证先在系统新增用户属性 + // 上报用户属性，需要保证先在系统新增用户属性 sdkExample.sendUserProfile(userUniqueId, appId); // 指定localTimeMs时间,即事件发生时间 diff --git a/datarangers-sdk-example/src/main/resources/application-pri.properties b/datarangers-sdk-example/src/main/resources/application-pri.properties deleted file mode 100644 index b493156..0000000 --- a/datarangers-sdk-example/src/main/resources/application-pri.properties +++ /dev/null @@ -1,10 +0,0 @@ -## Http\u8BF7\u6C42Header\u5185\u5BB9,Host\u4E3A\u5FC5\u987B -datarangers.sdk.mode=http -datarangers.sdk.headers.Host=${SDK_HOST} - -# batch -#datarangers.sdk.sendBatch=true -#datarangers.sdk.batchSize=16 -#datarangers.sdk.waitTimeMs=100 - - diff --git a/datarangers-sdk-example/src/main/resources/application-prifile.properties b/datarangers-sdk-example/src/main/resources/application-prifile.properties new file mode 100644 index 0000000..6b8b198 --- /dev/null +++ b/datarangers-sdk-example/src/main/resources/application-prifile.properties @@ -0,0 +1,11 @@ +# \u4F7F\u7528file\u4E0A\u62A5\u6A21\u5F0F\uFF0C\u9700\u8981\u914D\u5408 loagent \u4E00\u8D77\u4F7F\u7528 +datarangers.sdk.mode=file +datarangers.sdk.headers.Host=${SDK_HOST} + +# \u6587\u4EF6\u8DEF\u5F84 +#datarangers.sdk.save=false +#datarangers.sdk.eventSavePath=logs/ +#datarangers.sdk.eventSaveName=datarangers.log +#datarangers.sdk.eventSaveMaxFileSize=-1 +#datarangers.sdk.eventFilePaths=event/logs/1/,event/logs/2/,event/logs/3/,event/logs/4/,event/logs/5/,event/logs/6/ + diff --git a/datarangers-sdk-example/src/main/resources/application-prihttp.properties b/datarangers-sdk-example/src/main/resources/application-prihttp.properties new file mode 100644 index 0000000..3373632 --- /dev/null +++ b/datarangers-sdk-example/src/main/resources/application-prihttp.properties @@ -0,0 +1,30 @@ +# \u4F7F\u7528 http \u4E0A\u62A5\u6A21\u5F0F +datarangers.sdk.mode=http +# \u670D\u52A1\u5668ip\u6216\u57DF\u540D +datarangers.sdk.domain=${SDK_DOMAIN} +# host +datarangers.sdk.headers.Host=${SDK_HOST} + +#\u5F02\u6B65\u7EBF\u7A0B\u6570\u91CF\uFF0C\u5F53\u5E76\u53D1\u4E0D\u591F\u7684\u65F6\u5019\u53EF\u4EE5\u8C03\u6574\u8BE5\u6570\u636E +#datarangers.sdk.threadCount=20 + +#[batch size] +#datarangers.sdk.sendBatch=true +#datarangers.sdk.batchSize=20 +#datarangers.sdk.waitTimeMs=100 + +#[http config] +# \u5355\u4F4D\u662F\u6BEB\u79D2 +#datarangers.sdk.httpConfig.requestTimeout=10000 +#datarangers.sdk.httpConfig.connectTimeout=10000 +#datarangers.sdk.httpConfig.socketTimeout=20000 + +# batch +#datarangers.sdk.sendBatch=true +#datarangers.sdk.batchSize=16 +#datarangers.sdk.waitTimeMs=100 + +# \u5355\u4F4D\u662Fs +#datarangers.sdk.httpConfig.keepAliveTimeout=30 + + diff --git a/datarangers-sdk-example/src/main/resources/application-prihttpbatch.properties b/datarangers-sdk-example/src/main/resources/application-prihttpbatch.properties new file mode 100644 index 0000000..9001534 --- /dev/null +++ b/datarangers-sdk-example/src/main/resources/application-prihttpbatch.properties @@ -0,0 +1,30 @@ +# \u4F7F\u7528 http \u4E0A\u62A5\u6A21\u5F0F\uFF0C\u4F7F\u7528\u6279\u91CF\u4E0A\u62A5\u7684\u65B9\u5F0F +datarangers.sdk.mode=http +# \u670D\u52A1\u5668ip\u6216\u57DF\u540D +datarangers.sdk.domain=${SDK_DOMAIN} +# host +datarangers.sdk.headers.Host=${SDK_HOST} + +# \u5F02\u6B65\u7EBF\u7A0B\u6570\u91CF\uFF0C\u5F53\u5E76\u53D1\u4E0D\u591F\u7684\u65F6\u5019\u53EF\u4EE5\u8C03\u6574\u8BE5\u6570\u636E +# datarangers.sdk.threadCount=20 + +#[batch size] +#datarangers.sdk.sendBatch=true +#datarangers.sdk.batchSize=16 +#datarangers.sdk.waitTimeMs=100 + +#[http config] +# \u5355\u4F4D\u662F\u6BEB\u79D2 +#datarangers.sdk.httpConfig.requestTimeout=10000 +#datarangers.sdk.httpConfig.connectTimeout=10000 +#datarangers.sdk.httpConfig.socketTimeout=20000 + +# \u4F7F\u7528 batch \u7684\u65B9\u5F0F +datarangers.sdk.sendBatch=true +#datarangers.sdk.batchSize=16 +#datarangers.sdk.waitTimeMs=100 + +# \u5355\u4F4D\u662Fs +#datarangers.sdk.httpConfig.keepAliveTimeout=30 + + diff --git a/datarangers-sdk-example/src/main/resources/application-prikafka.properties b/datarangers-sdk-example/src/main/resources/application-prikafka.properties new file mode 100644 index 0000000..f850034 --- /dev/null +++ b/datarangers-sdk-example/src/main/resources/application-prikafka.properties @@ -0,0 +1,7 @@ +# \u4F7F\u7528kafka\u4E0A\u62A5\u7684\u6A21\u5F0F +datarangers.sdk.mode=kafka +datarangers.sdk.kafka.bootstrapServers={ip1}:9192,{ip2}:9192 + +# kafka producer\u7684 properties\u53EF\u4EE5\u5728\u8FD9\u91CC\u8FDB\u884C\u914D\u7F6E +#datarangers.sdk.kafka.properties.retries=3 + diff --git a/datarangers-sdk-example/src/main/resources/application.properties b/datarangers-sdk-example/src/main/resources/application.properties new file mode 100644 index 0000000..e45f01e --- /dev/null +++ b/datarangers-sdk-example/src/main/resources/application.properties @@ -0,0 +1,11 @@ +#\u65E5\u5FD7\u914D\u7F6E +logging.level.root=debug +logging.pattern.console=%d{HH:mm:ss.SSS} [%t] %-5level %logger{36}:%L - %msg%n + +datarangers.sdk.enable=true + +# \u4F7F\u7528\u54EA\u4E2A\u914D\u7F6E\u6587\u4EF6 +spring.profiles.active=${PROFILE} + + + diff --git a/datarangers-sdk-starter/src/main/resources/application.properties b/datarangers-sdk-starter/src/main/resources/application.properties new file mode 100644 index 0000000..f9a1514 --- /dev/null +++ b/datarangers-sdk-starter/src/main/resources/application.properties @@ -0,0 +1,41 @@ +#logging.level.root=info +# \u670D\u52A1\u5668ip\u6216\u57DF\u540D +#datarangers.sdk.domain=${HOST_IP} + +## Http\u8BF7\u6C42Header\u5185\u5BB9,Host\u4E3A\u5FC5\u987B +# datarangers.sdk.headers.Host=${VPC_HOST} +# \u4E3Atrue\u4FDD\u8BC1Bean\u80FD\u591F\u6CE8\u5165 +#datarangers.sdk.enable=true +#datarangers.sdk.save=true +#datarangers.sdk.eventSavePath=logs/ +#datarangers.sdk.eventSaveName=datarangers.log +#datarangers.sdk.eventSaveMaxFileSize=1 +#datarangers.sdk.eventFilePaths=logs/1/,logs/2/,logs/3/,logs/4/,logs/5/,logs/6/ +# \u6D88\u8D39\u8005\u7EBF\u7A0B\u6570\u91CF +#datarangers.sdk.threadCount=4 +# \u961F\u5217\u957F\u5EA6 +#datarangers.sdk.queueSize=1024000 + +# http \u8D85\u65F6\u914D\u7F6E +# datarangers.sdk.httpConfig.requestTimeout=10000 +# datarangers.sdk.httpConfig.connectTimeout=10000 +# datarangers.sdk.httpConfig.socketTimeout=20000 + +# client\u662F\u5426\u9700\u8981\u8FDB\u884Cssl\u8BC1\u4E66\u8BA4\u8BC1\uFF0C\u9ED8\u8BA4\u4E3Afalse\uFF0Cfalse\u8868\u793A\u9700\u8981\u8FDB\u884C\u8BC1\u4E66\u8BA4\u8BC1 +# datarangers.sdk.httpConfig.trustDisable=false + +# \u81EA\u5B9A\u4E49\u8BC1\u4E66\u8DEF\u5F84\u548C\u5BC6\u7801\uFF0Cfalse\u8868\u793A\u4F7F\u7528jdk\u81EA\u8EAB\u7684\u9ED8\u8BA4\u8DEF\u5F84 +# datarangers.sdk.httpConfig.customKeyTrustEnable=false + +# \u914D\u7F6E\u8BC1\u4E66 +# datarangers.sdk.httpConfig.keyMaterialPath=xxx +# datarangers.sdk.httpConfig.keyPassword=xxx +# datarangers.sdk.httpConfig.storePassword=xxx +# datarangers.sdk.httpConfig.trustMaterialPath=xxx + +# self for selfTrustStrategy, default is all +# datarangers.sdk.httpConfig.trustStrategy=xxx + +# appKeys \u914D\u7F6E +# datarangers.sdk.appKeys.${appId}=${appKey} + From 65a89536f47f9aff1ffece1ec486416d97190662 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Sat, 8 Oct 2022 21:55:09 +0800 Subject: [PATCH 13/21] feature: remove unused config --- .../datarangers/asynccollector/Consumer.java | 8 +- .../com/datarangers/collector/Collector.java | 169 ++++++++++-------- .../DataRangersSDKConfigProperties.java | 80 ++++----- .../com/datarangers/config/EventConfig.java | 2 - .../com/datarangers/config/OpenapiConfig.java | 5 + 5 files changed, 134 insertions(+), 130 deletions(-) diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java index 5961d95..8210d27 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/asynccollector/Consumer.java @@ -7,8 +7,8 @@ package com.datarangers.asynccollector; import com.datarangers.config.DataRangersSDKConfigProperties; -import com.datarangers.config.EventConfig; import com.datarangers.config.RangersJSONConfig; +import com.datarangers.config.SdkMode; import com.datarangers.logger.RangersLoggerWriterPool; import com.datarangers.message.AppMessage; import com.datarangers.message.Message; @@ -93,7 +93,7 @@ private void write() throws Exception { @Override public void run() { try { - if (EventConfig.saveFlag) { + if (SdkMode.FILE == sdkConfigProperties.getMode()) { write(); } else if (sdkConfigProperties.isSendBatch()) { sendBatch(); @@ -116,7 +116,7 @@ public void flush() { while (message != null) { count++; message = collectorContainer.handleMessage(message); - if (EventConfig.saveFlag) { + if (SdkMode.FILE == sdkConfigProperties.getMode()) { doWrite(message); } else { doSend(message); @@ -135,7 +135,7 @@ public void flush() { public void flush(Message message) { message = collectorContainer.handleMessage(message); - if (EventConfig.saveFlag) { + if (SdkMode.FILE == sdkConfigProperties.getMode()) { doWrite(message); } else { doSend(message); diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java index 6b38a0d..7d48f81 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java @@ -41,7 +41,7 @@ public abstract class Collector implements EventCollector { private String appType; public static final Logger logger = LoggerFactory.getLogger("DatarangersLog"); - public static ExecutorService httpRequestPool = null; + public static ExecutorService executorService = null; public static ScheduledExecutorService scheduled = null; public static CollectorContainer collectorContainer; private boolean enable; @@ -87,11 +87,11 @@ public void setConsumer(Consumer consumer) { } - private void initKafkaProducer() { - logger.info("init kafka producer"); + private void initModeKafka() { if (SdkMode.KAFKA != this.properties.getMode()) { return; } + logger.info("init kafka producer"); // 设置过了就不需要再自己创建 if (kafkaProducer != null) { return; @@ -209,15 +209,23 @@ private void init() { if (!IS_INIT) { initLogger(); initCommon(); - initKafkaProducer(); + initSdkMode(); initConsumer(); initHook(); IS_INIT = true; + System.out.println(String.format("sdk config: %s\r\n", this.properties)); + logger.info("sdk config: %s\r\n", this.properties); } } } } + private void initSdkMode() { + initModeFile(); + initModeHttp(); + initModeKafka(); + } + /** * 日志pool初始化 */ @@ -239,95 +247,100 @@ private void initLogger() { * eventConfig,httpclient,EventConfig 初始化 */ private void initCommon() { - HttpConfig httpConfig = properties.getHttpConfig(); - HttpClient httpClient = properties.getCustomHttpClient(); - Callback callback = properties.getCallback(); - int httpTimeOut = properties.getHttpTimeout(); - - //EventConfig配置 - EventConfig.saveFlag = SdkMode.FILE == properties.getMode(); - EventConfig.sendFlag = SdkMode.HTTP == properties.getMode(); - EventConfig.setUrl(properties.getDomain()); - - if (EventConfig.sendFlag) { - if (httpConfig.getMaxPerRoute() < properties.getCorePoolSize()) { - httpConfig.setMaxPerRoute(properties.getCorePoolSize()); - } - if (httpConfig.getMaxTotal() < httpConfig.getMaxPerRoute()) { - httpConfig.setMaxTotal(httpConfig.getMaxPerRoute()); - } - // 老版本配置做兼容 - httpConfig.initTimeOut(httpTimeOut); - //httpclient 初始化 - HttpUtils.createHttpClient(httpConfig, httpClient, callback); - - //EventConfig 初始化 - if (EventConfig.SEND_HEADER == null) { - EventConfig.SEND_HEADER = properties.getHeaders(); - EventConfig.SEND_HEADER.put("User-Agent", "DataRangers Java SDK"); - EventConfig.SEND_HEADER.put("Content-Type", "application/json"); - List
headerList = new ArrayList<>(); - EventConfig.SEND_HEADER - .forEach((key, value) -> headerList.add(new BasicHeader(key, value))); - EventConfig.headers = headerList.toArray(new Header[0]); - } - } - } - - /** - * 初始化消费者, httpRequestPool, 日志记录和清理任务 - */ - private void initConsumer() { - logger.info("init consumer"); - int threadCount = this.properties.getCorePoolSize(); - CollectorQueue userQueue = this.properties.getUserQueue(); - - if (EventConfig.saveFlag) { - threadCount = 1; - logger.info("Start LogAgent Mode"); - } else { - logger.info("Start Http Mode"); - } - // 如果客户自定义了queue，则需要替换为客户自定义queue，否则使用默认的队列 + CollectorQueue userQueue = this.properties.getUserQueue(); if (userQueue == null) { collectorContainer = new CollectorContainer( RangersCollectorQueue.getInstance(this.properties.getQueueSize())); } else { collectorContainer = new CollectorContainer(userQueue); } + } + /** + * 初始化消费者, httpRequestPool, 日志记录和清理任务 + */ + private void initConsumer() { + logger.info("init consumer"); boolean isSync = this.properties.isSync(); - boolean hasConsumer = this.properties.isHasConsumer(); - boolean hasProducer = this.properties.isHasProducer(); - String eventSavePath = this.properties.getEventSavePath(); - List eventFilePaths = properties.getEventFilePaths(); - String eventSaveName = this.properties.getEventSaveName(); - int eventSaveMaxDays = this.properties.getEventSaveMaxDays(); - // 设置同步发送的consumer，队列满的时候使用 + + // 同步设置 if (isSync) { setConsumer(new Consumer(Collector.collectorContainer, this.properties)); + return; } + // 异步起多个消费者 - if (!isSync && hasConsumer && httpRequestPool == null) { - httpRequestPool = Executors.newFixedThreadPool(this.properties.getCorePoolSize()); + if (executorService == null) { + int threadCount = this.properties.getThreadCount(); + executorService = Executors.newFixedThreadPool(threadCount); + + // 创建consumer for (int i = 0; i < threadCount; i++) { //必须全部消费同一个队列 - httpRequestPool.execute(new Consumer(collectorContainer, properties)); + executorService.execute(new Consumer(collectorContainer, properties)); } } - if ((!isSync) && hasProducer) { - //定时记录日志的条数 - scheduled = Executors.newSingleThreadScheduledExecutor(); - scheduled - .scheduleAtFixedRate(new CollectorCounter(eventSavePath), 0, 2, TimeUnit.MINUTES); - if (EventConfig.saveFlag) { - // 清理日志文件定时任务, 每隔12小时清理一次 - scheduled.scheduleAtFixedRate( - new RangersFileCleaner(eventFilePaths, eventSaveName, eventSaveMaxDays), - 0, 12, TimeUnit.HOURS); - logger.info("Start DataRangers Cleaner/Record Thread"); - } + } + + private void initModeHttp() { + if(SdkMode.HTTP != this.properties.getMode()){ + return; + } + HttpConfig httpConfig = properties.getHttpConfig(); + HttpClient httpClient = properties.getCustomHttpClient(); + Callback callback = properties.getCallback(); + int httpTimeOut = properties.getHttpTimeout(); + + //EventConfig配置 + EventConfig.setUrl(properties.getDomain()); + if (httpConfig.getMaxPerRoute() < properties.getThreadCount()) { + httpConfig.setMaxPerRoute(properties.getThreadCount()); + } + if (httpConfig.getMaxTotal() < httpConfig.getMaxPerRoute()) { + httpConfig.setMaxTotal(httpConfig.getMaxPerRoute()); + } + // 老版本配置做兼容 + httpConfig.initTimeOut(httpTimeOut); + //httpclient 初始化 + HttpUtils.createHttpClient(httpConfig, httpClient, callback); + + //EventConfig 初始化 + if (EventConfig.SEND_HEADER == null) { + EventConfig.SEND_HEADER = properties.getHeaders(); + EventConfig.SEND_HEADER.put("User-Agent", "DataRangers Java SDK"); + EventConfig.SEND_HEADER.put("Content-Type", "application/json"); + List
headerList = new ArrayList<>(); + EventConfig.SEND_HEADER + .forEach((key, value) -> headerList.add(new BasicHeader(key, value))); + EventConfig.headers = headerList.toArray(new Header[0]); + } + } + + private void initModeFile() { + if(SdkMode.FILE != this.properties.getMode()){ + return; + } + // thread 设置为1 + this.properties.setThreadCount(1); + + String eventSavePath = this.properties.getEventSavePath(); + List eventFilePaths = properties.getEventFilePaths(); + String eventSaveName = this.properties.getEventSaveName(); + int eventSaveMaxDays = this.properties.getEventSaveMaxDays(); + + //定时记录日志的条数 + scheduled = Executors.newSingleThreadScheduledExecutor(); + + scheduled + .scheduleAtFixedRate(new CollectorCounter(eventSavePath), 0, 2, TimeUnit.MINUTES); + + if (eventSaveMaxDays > 0) { + // 清理日志文件定时任务, 每隔12小时清理一次 + scheduled.scheduleAtFixedRate( + new RangersFileCleaner(eventFilePaths, eventSaveName, eventSaveMaxDays), + 0, 12, TimeUnit.HOURS); + logger.info("Start DataRangers Cleaner/Record Thread"); } } @@ -338,8 +351,8 @@ private void initHook() { logger.info("init hook"); Runtime.getRuntime().addShutdownHook(new Thread( () -> { - if (Collector.httpRequestPool != null) { - Collector.httpRequestPool.shutdown(); + if (Collector.executorService != null) { + Collector.executorService.shutdown(); } new Consumer(Collector.collectorContainer, properties).flush(); diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java index b02d6a4..f6983a3 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java @@ -18,10 +18,10 @@ * @author hTangle */ public class DataRangersSDKConfigProperties { - + public boolean enable = true; public Map headers; public String domain; - public int corePoolSize = 4; + private int threadCount = 4; /** * 该配置过期，请使用httpConfig的配置 @@ -40,27 +40,30 @@ public class DataRangersSDKConfigProperties { */ @Deprecated public boolean save = false; + public int queueSize = 10240; - public boolean send = true; + /** + * batch批量配置 + */ private boolean sendBatch = false; private int batchSize = 20; private int waitTimeMs = 100; - public boolean enable = true; - + /** + * file模式 相关配置 + */ public String eventSavePath = "logs/"; public List eventFilePaths; public String eventSaveName = "datarangers.log"; public int eventSaveMaxFileSize = 100; + // 日志清理时间 public int eventSaveMaxDays = 5; public CollectorQueue userQueue; - public boolean hasConsumer = true; - public boolean hasProducer = true; - + // http相关配置 private HttpConfig httpConfig = new HttpConfig(); private HttpClient customHttpClient; @@ -161,24 +164,6 @@ public void setCallback(Callback callback) { this.callback = callback; } - public boolean isHasConsumer() { - return hasConsumer; - } - - public DataRangersSDKConfigProperties setHasConsumer(boolean hasConsumer) { - this.hasConsumer = hasConsumer; - return this; - } - - public boolean isHasProducer() { - return hasProducer; - } - - public DataRangersSDKConfigProperties setHasProducer(boolean hasProducer) { - this.hasProducer = hasProducer; - return this; - } - public int getHttpTimeout() { return httpTimeout; } @@ -237,15 +222,6 @@ public DataRangersSDKConfigProperties setEventSaveMaxFileSize(int eventSaveMaxFi return this; } - public boolean isSend() { - return send; - } - - public DataRangersSDKConfigProperties setSend(boolean send) { - this.send = send; - return this; - } - public int getQueueSize() { return queueSize; } @@ -254,12 +230,12 @@ public void setQueueSize(int queueSize) { this.queueSize = queueSize; } - public int getCorePoolSize() { - return corePoolSize; + public int getThreadCount() { + return threadCount; } - public void setCorePoolSize(int corePoolSize) { - this.corePoolSize = corePoolSize; + public void setThreadCount(int threadCount) { + this.threadCount = threadCount; } public Map getHeaders() { @@ -366,13 +342,25 @@ public void setAppKeys(Map appKeys) { @Override public String toString() { - return " domain:" + domain + " corePoolSize:" + corePoolSize + " httpTimout:" + httpTimeout + - " timeZone:" + timeZone + " timeOffset:" + timeOffset + " save:" + save + " queueSize:" + queueSize + - " send:" + send + " sendBatch:" + sendBatch + " batchSize:" + batchSize + " waitTimeMs:" + waitTimeMs + - " enable:" + enable + " eventSavePath:" + eventSavePath + " eventSaveName:" + eventSaveName + - " eventSaveMaxFileSize:" + eventSaveMaxFileSize + " eventSaveMaxDays:" + eventSaveMaxDays + - " userQueue:" + userQueue + " hasConsumer:" + hasConsumer + " hasProducer:" + hasProducer + - " env" + env + " sync:" + sync + " mode:" + mode + " kafka:" + kafka + " httpConfig" + httpConfig; + StringBuilder sb = new StringBuilder(); + sb.append(String.format("enable: %s \r\n", enable)); + sb.append(String.format("env: %s \r\n", env)); + sb.append(String.format("sync: %s \r\n", sync)); + sb.append(String.format("domain: %s \r\n", domain)); + sb.append(String.format("threadCount: %s \r\n", threadCount)); + sb.append(String.format("sdkMode: %s \r\n", mode)); + sb.append(String.format("queueSize: %s \r\n", queueSize)); + sb.append(String.format("sendBatch: %s \r\n", sendBatch)); + sb.append(String.format("batchSize: %s \r\n", batchSize)); + sb.append(String.format("waitTimeMs: %s \r\n", waitTimeMs)); + sb.append(String.format("eventSavePath: %s \r\n", eventSavePath)); + sb.append(String.format("eventSaveName: %s \r\n", eventSaveName)); + sb.append(String.format("eventSaveMaxFileSize: %s \r\n", eventSaveMaxFileSize)); + sb.append(String.format("eventSaveMaxDays: %s \r\n", eventSaveMaxDays)); + sb.append(String.format("httpConfig: %s \r\n", httpConfig)); + sb.append(String.format("openapiConfig: %s \r\n", openapiConfig)); + sb.append(String.format("kafka: %s \r\n", kafka)); + return sb.toString(); } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java index 9244bfe..6c5dcbe 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/EventConfig.java @@ -15,8 +15,6 @@ public class EventConfig { public static String appListUrl; public static Header[] headers; public static Map SEND_HEADER; - public static boolean saveFlag = false; - public static boolean sendFlag = true; public static void setUrl(String url) { setAppUrl(url + Constants.APP_LOG_PATH); diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/OpenapiConfig.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/OpenapiConfig.java index c257657..1a02cb6 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/OpenapiConfig.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/OpenapiConfig.java @@ -33,4 +33,9 @@ public String getSk() { public void setSk(String sk) { this.sk = sk; } + + @Override + public String toString(){ + return String.format("{\"domain\": %s, \"ak\"=\"xxxxxx\", \"sk\"=\"xxxxxx\"}", domain); + } } From 885eaef9d82565d0d364d28de6f496709178c57c Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Mon, 10 Oct 2022 10:52:57 +0800 Subject: [PATCH 14/21] feature: optimize code and example --- SDKTest.iml | 2 - .../com/datarangers/collector/Collector.java | 4 +- .../DataRangersSDKConfigProperties.java | 16 ++++---- .../com/datarangers/config/HttpConfig.java | 37 ++++++++++++++----- .../com/datarangers/config/OpenapiConfig.java | 4 +- .../main/java/com/datarangers/util/Tools.java | 27 ++++++++++++++ .../example/BatchPriSdkExample.java | 2 +- .../example/FailCallbackExample.java | 2 +- .../example/HttpPriSdkExample.java | 2 +- .../example/KafkaPriSdkExample.java | 8 +++- .../datarangers/example/SaasSdkExample.java | 4 +- .../resources/application-prifile.properties | 10 +++-- .../resources/application-prihttp.properties | 13 +------ .../application-prihttpbatch.properties | 14 +++---- .../resources/application-prikafka.properties | 1 + .../resources/application-saas.properties | 12 +++++- 16 files changed, 106 insertions(+), 52 deletions(-) delete mode 100644 SDKTest.iml create mode 100644 datarangers-sdk-core/src/main/java/com/datarangers/util/Tools.java diff --git a/SDKTest.iml b/SDKTest.iml deleted file mode 100644 index 78b2cc5..0000000 --- a/SDKTest.iml +++ /dev/null @@ -1,2 +0,0 @@ - - \ No newline at end of file diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java index 7d48f81..6d100eb 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java @@ -52,8 +52,8 @@ public abstract class Collector implements EventCollector { private static volatile Boolean IS_INIT = false; public Collector(String appType, DataRangersSDKConfigProperties properties, Callback cb) { - logger.info("sdk config properties: {}", properties); - System.out.println("sdk config properties: " + properties.toString()); + logger.info("sdk config properties: \r\n{}", properties); + System.out.println("sdk config properties: \r\n" + properties.toString()); this.appType = appType; this.enable = properties.isEnable(); this.properties = properties; diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java index f6983a3..fd32b7f 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/DataRangersSDKConfigProperties.java @@ -21,13 +21,13 @@ public class DataRangersSDKConfigProperties { public boolean enable = true; public Map headers; public String domain; - private int threadCount = 4; + private int threadCount = 20; /** * 该配置过期，请使用httpConfig的配置 */ @Deprecated - public int httpTimeout = 500; + public int httpTimeout = 10000; public String timeZone = "+8"; public ZoneOffset timeOffset = null; @@ -59,7 +59,7 @@ public class DataRangersSDKConfigProperties { public int eventSaveMaxFileSize = 100; // 日志清理时间 - public int eventSaveMaxDays = 5; + public int eventSaveMaxDays = -1; public CollectorQueue userQueue; @@ -344,22 +344,24 @@ public void setAppKeys(Map appKeys) { public String toString() { StringBuilder sb = new StringBuilder(); sb.append(String.format("enable: %s \r\n", enable)); - sb.append(String.format("env: %s \r\n", env)); + sb.append(String.format("env: %s \r\n", getMessageEnv())); sb.append(String.format("sync: %s \r\n", sync)); + sb.append(String.format("sdkMode: %s \r\n", mode)); sb.append(String.format("domain: %s \r\n", domain)); + sb.append(String.format("headers: %s \r\n", headers)); sb.append(String.format("threadCount: %s \r\n", threadCount)); - sb.append(String.format("sdkMode: %s \r\n", mode)); sb.append(String.format("queueSize: %s \r\n", queueSize)); sb.append(String.format("sendBatch: %s \r\n", sendBatch)); sb.append(String.format("batchSize: %s \r\n", batchSize)); sb.append(String.format("waitTimeMs: %s \r\n", waitTimeMs)); + sb.append(String.format("httpConfig: %s \r\n", httpConfig)); sb.append(String.format("eventSavePath: %s \r\n", eventSavePath)); sb.append(String.format("eventSaveName: %s \r\n", eventSaveName)); + sb.append(String.format("eventFilePaths: %s \r\n", eventFilePaths)); sb.append(String.format("eventSaveMaxFileSize: %s \r\n", eventSaveMaxFileSize)); sb.append(String.format("eventSaveMaxDays: %s \r\n", eventSaveMaxDays)); - sb.append(String.format("httpConfig: %s \r\n", httpConfig)); sb.append(String.format("openapiConfig: %s \r\n", openapiConfig)); - sb.append(String.format("kafka: %s \r\n", kafka)); + sb.append(String.format("kafka: %s", kafka)); return sb.toString(); } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java index 2f4bb13..67cf647 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/HttpConfig.java @@ -7,6 +7,8 @@ package com.datarangers.config; +import com.datarangers.util.Tools; + /** * @author bytedance * @date 2020/12/2 20:02 @@ -18,10 +20,14 @@ public class HttpConfig { private Integer maxTotal = 1000; private Integer maxPerRoute = 100; - private Integer requestTimeout; - private Integer connectTimeout; - private Integer socketTimeout; - private Integer keepAliveTimeout; + private Integer requestTimeout = 10 * 1000; + private Integer connectTimeout = 10 * 1000; + private Integer socketTimeout = 20 * 1000; + + /** + * 单位是s，默认1分钟 + */ + private Integer keepAliveTimeout = 30; /** * 是否需要自定义配置key, store 路径和密码 @@ -134,7 +140,7 @@ public void initTimeOut(int timeBase) { if(keepAliveTimeout == null) { // 3分钟 - keepAliveTimeout = 3 * 60; + keepAliveTimeout = 30; } } @@ -164,10 +170,21 @@ public void setKeepAliveTimeout(Integer keepAliveTimeout) { @Override public String toString() { - return " requestTimeout:" + requestTimeout + "connectTimeout:" + connectTimeout + - " socketTimeout:" + socketTimeout + " keepAliveTimeout:" + keepAliveTimeout + - " maxTotal:" + maxTotal + " maxPerRoute:" + maxPerRoute + - " keyMaterialPath:" + keyMaterialPath + " keyPassword:" + keyPassword + - " storePassword:" + storePassword + " trustMaterialPath:" + trustMaterialPath; + StringBuilder sb = new StringBuilder(); + sb.append("{"); + + sb.append(String.format("\"requestTimeout\": %s,", requestTimeout)); + sb.append(String.format("\"connectTimeout\": %s,", connectTimeout)); + sb.append(String.format("\"socketTimeout\": %s,", socketTimeout)); + sb.append(String.format("\"keepAliveTimeout\": %s,", keepAliveTimeout)); + sb.append(String.format("\"maxTotal\": %s,", maxTotal)); + sb.append(String.format("\"maxPerRoute\": %s,", maxPerRoute)); + sb.append(String.format("\"keyMaterialPath\": \"%s\",", keyMaterialPath)); + sb.append(String.format("\"keyPassword\": \"%s\",", Tools.passwordMask(keyPassword))); + sb.append(String.format("\"trustMaterialPath\": \"%s\",", trustMaterialPath)); + sb.append(String.format("\"storePassword\": \"%s\"", Tools.passwordMask(storePassword))); + + sb.append("}"); + return sb.toString(); } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/OpenapiConfig.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/OpenapiConfig.java index 1a02cb6..74cdb87 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/OpenapiConfig.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/OpenapiConfig.java @@ -1,5 +1,7 @@ package com.datarangers.config; +import com.datarangers.util.Tools; + /** * @Author zhangpeng.spin@bytedance.com * @Date 2021-07-23 @@ -36,6 +38,6 @@ public void setSk(String sk) { @Override public String toString(){ - return String.format("{\"domain\": %s, \"ak\"=\"xxxxxx\", \"sk\"=\"xxxxxx\"}", domain); + return String.format("{\"domain\": %s, \"ak\"=\"%s\", \"sk\"=\"%s\"}", domain, Tools.passwordMask(ak), Tools.passwordMask(sk)); } } diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/util/Tools.java b/datarangers-sdk-core/src/main/java/com/datarangers/util/Tools.java new file mode 100644 index 0000000..f95bfab --- /dev/null +++ b/datarangers-sdk-core/src/main/java/com/datarangers/util/Tools.java @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.util; + +/** +// * @Author zhangpeng.spin + * @Date 2022/10/9 + */ +public class Tools { + private Tools() { + } + + public static String passwordMask(String password) { + if (password == null) { + return null; + } + return "******"; + } +} diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java index 0465aa9..ca22d28 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/BatchPriSdkExample.java @@ -31,7 +31,7 @@ public class BatchPriSdkExample extends AbstractSdkExample { // 设置模式 properties.setMode(SdkMode.HTTP); - // 设置domain和host + // 设置domain和host，这里注意替换成真实的参数 properties.setDomain(System.getenv("SDK_DOMAIN")); properties.getHeaders().put("HOST", System.getenv("SDK_HOST")); diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java index 403c04e..074711a 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/FailCallbackExample.java @@ -30,7 +30,7 @@ public abstract class FailCallbackExample extends AbstractSdkExample { // 设置模式 properties.setMode(SdkMode.HTTP); - // 设置domain和host + // 设置domain和host，这里注意替换成真实的参数 properties.setDomain(System.getenv("SDK_DOMAIN")); properties.getHeaders().put("HOST", System.getenv("SDK_HOST")); diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java index 3c851d5..07dc4c6 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkExample.java @@ -31,7 +31,7 @@ public class HttpPriSdkExample extends AbstractSdkExample { // 设置模式 properties.setMode(SdkMode.HTTP); - // 设置domain和host + // 设置domain和host，这里注意替换成真实的参数 properties.setDomain(System.getenv("SDK_DOMAIN")); properties.getHeaders().put("HOST", System.getenv("SDK_HOST")); diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java index 0475b96..e3f7827 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java @@ -37,8 +37,12 @@ public class KafkaPriSdkExample extends AbstractSdkExample { KafkaConfig kafkaConfig = new KafkaConfig(); properties.setKafka(kafkaConfig); - // 设置 BootstrapServers, etc: 127.0.0.1:9192,localhost:9192 - kafkaConfig.setBootstrapServers(System.getenv("SDK_KAFKA_BOOTSTRAP_SERVERS")); + // 设置 BootstrapServers, etc: 127.0.0.1:9192,localhost:9192，这里注意替换成真实的参数 + String bootstrapServers = System.getenv("SDK_KAFKA_BOOTSTRAP_SERVERS"); + if(bootstrapServers == null){ + throw new IllegalArgumentException("bootstrapServers can not be empty"); + } + kafkaConfig.setBootstrapServers(bootstrapServers); // 如果需要设置其他属性，可以进行设置 // Map kafkaProperties = new HashMap<>(); diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java index 585b23e..94684cb 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/SaasSdkExample.java @@ -34,10 +34,10 @@ public class SaasSdkExample extends AbstractSdkExample { // 设置domain和appKey, 不需要设置HOST properties.setDomain("https://mcs.ctobsnssdk.com"); - // 可以设置多个app + // 可以设置多个app，这里注意替换成真实的参数 properties.getAppKeys().put(Integer.valueOf(System.getenv("SDK_APP_1")), System.getenv("SDK_APP_KEY_1")); - // 设置openapi domain, AK,SK + // 设置openapi domain, AK,SK，这里注意替换成真实的参数 properties.getOpenapiConfig().setDomain("https://analytics.volcengineapi.com"); properties.getOpenapiConfig().setAk(System.getenv("OPENAPI_AK")); properties.getOpenapiConfig().setSk(System.getenv("OPENAPI_SK")); diff --git a/datarangers-sdk-example/src/main/resources/application-prifile.properties b/datarangers-sdk-example/src/main/resources/application-prifile.properties index 6b8b198..c3cf1d0 100644 --- a/datarangers-sdk-example/src/main/resources/application-prifile.properties +++ b/datarangers-sdk-example/src/main/resources/application-prifile.properties @@ -1,11 +1,15 @@ +# \u79C1\u6709\u5316\u914D\u7F6Eexample # \u4F7F\u7528file\u4E0A\u62A5\u6A21\u5F0F\uFF0C\u9700\u8981\u914D\u5408 loagent \u4E00\u8D77\u4F7F\u7528 datarangers.sdk.mode=file -datarangers.sdk.headers.Host=${SDK_HOST} # \u6587\u4EF6\u8DEF\u5F84 -#datarangers.sdk.save=false #datarangers.sdk.eventSavePath=logs/ #datarangers.sdk.eventSaveName=datarangers.log -#datarangers.sdk.eventSaveMaxFileSize=-1 +# \u5355\u4F4D\u662FM +#datarangers.sdk.eventSaveMaxFileSize=100 +# \u5982\u679C\u6CA1\u6709\u914D\u7F6EeventFilePaths\uFF0C\u90A3\u4E48\u4F1A\u628A\u65E5\u5FD7\u6587\u4EF6\u653E\u5230eventSavePath\u76EE\u5F55\u4E0B #datarangers.sdk.eventFilePaths=event/logs/1/,event/logs/2/,event/logs/3/,event/logs/4/,event/logs/5/,event/logs/6/ +# \u6587\u4EF6\u6700\u5927\u4FDD\u7559\u65F6\u95F4\uFF0C\u9ED8\u8BA4\u662F-1\uFF0C\u4E00\u76F4\u4FDD\u7559 +#datarangers.sdk.eventSaveMaxDays=-1 + diff --git a/datarangers-sdk-example/src/main/resources/application-prihttp.properties b/datarangers-sdk-example/src/main/resources/application-prihttp.properties index 3373632..853ad37 100644 --- a/datarangers-sdk-example/src/main/resources/application-prihttp.properties +++ b/datarangers-sdk-example/src/main/resources/application-prihttp.properties @@ -1,29 +1,20 @@ +# \u79C1\u6709\u5316\u914D\u7F6Eexample # \u4F7F\u7528 http \u4E0A\u62A5\u6A21\u5F0F datarangers.sdk.mode=http # \u670D\u52A1\u5668ip\u6216\u57DF\u540D datarangers.sdk.domain=${SDK_DOMAIN} -# host +# host\uFF0C\u79C1\u6709\u5316\u73AF\u5883Host\u7684\u914D\u7F6E\u5728\u5B89\u88C5\u90E8\u7F72\u7684\u90A3\u53F0\u673A\u5668\u4E0A\uFF0C\u67E5\u770B/home/datarangers/DataRangersDeploy/conf_rangers.yml\u4E2D\u914D\u7F6E\u9879sdk.report.host datarangers.sdk.headers.Host=${SDK_HOST} #\u5F02\u6B65\u7EBF\u7A0B\u6570\u91CF\uFF0C\u5F53\u5E76\u53D1\u4E0D\u591F\u7684\u65F6\u5019\u53EF\u4EE5\u8C03\u6574\u8BE5\u6570\u636E #datarangers.sdk.threadCount=20 -#[batch size] -#datarangers.sdk.sendBatch=true -#datarangers.sdk.batchSize=20 -#datarangers.sdk.waitTimeMs=100 - #[http config] # \u5355\u4F4D\u662F\u6BEB\u79D2 #datarangers.sdk.httpConfig.requestTimeout=10000 #datarangers.sdk.httpConfig.connectTimeout=10000 #datarangers.sdk.httpConfig.socketTimeout=20000 -# batch -#datarangers.sdk.sendBatch=true -#datarangers.sdk.batchSize=16 -#datarangers.sdk.waitTimeMs=100 - # \u5355\u4F4D\u662Fs #datarangers.sdk.httpConfig.keepAliveTimeout=30 diff --git a/datarangers-sdk-example/src/main/resources/application-prihttpbatch.properties b/datarangers-sdk-example/src/main/resources/application-prihttpbatch.properties index 9001534..e1436cd 100644 --- a/datarangers-sdk-example/src/main/resources/application-prihttpbatch.properties +++ b/datarangers-sdk-example/src/main/resources/application-prihttpbatch.properties @@ -1,3 +1,4 @@ +# \u79C1\u6709\u5316\u914D\u7F6Eexample # \u4F7F\u7528 http \u4E0A\u62A5\u6A21\u5F0F\uFF0C\u4F7F\u7528\u6279\u91CF\u4E0A\u62A5\u7684\u65B9\u5F0F datarangers.sdk.mode=http # \u670D\u52A1\u5668ip\u6216\u57DF\u540D @@ -8,23 +9,20 @@ datarangers.sdk.headers.Host=${SDK_HOST} # \u5F02\u6B65\u7EBF\u7A0B\u6570\u91CF\uFF0C\u5F53\u5E76\u53D1\u4E0D\u591F\u7684\u65F6\u5019\u53EF\u4EE5\u8C03\u6574\u8BE5\u6570\u636E # datarangers.sdk.threadCount=20 -#[batch size] -#datarangers.sdk.sendBatch=true -#datarangers.sdk.batchSize=16 -#datarangers.sdk.waitTimeMs=100 - #[http config] # \u5355\u4F4D\u662F\u6BEB\u79D2 #datarangers.sdk.httpConfig.requestTimeout=10000 #datarangers.sdk.httpConfig.connectTimeout=10000 #datarangers.sdk.httpConfig.socketTimeout=20000 +# \u5355\u4F4D\u662Fs +#datarangers.sdk.httpConfig.keepAliveTimeout=30 +# [batch] # \u4F7F\u7528 batch \u7684\u65B9\u5F0F datarangers.sdk.sendBatch=true -#datarangers.sdk.batchSize=16 +#datarangers.sdk.batchSize=20 #datarangers.sdk.waitTimeMs=100 -# \u5355\u4F4D\u662Fs -#datarangers.sdk.httpConfig.keepAliveTimeout=30 + diff --git a/datarangers-sdk-example/src/main/resources/application-prikafka.properties b/datarangers-sdk-example/src/main/resources/application-prikafka.properties index f850034..8aa015f 100644 --- a/datarangers-sdk-example/src/main/resources/application-prikafka.properties +++ b/datarangers-sdk-example/src/main/resources/application-prikafka.properties @@ -1,3 +1,4 @@ +# \u79C1\u6709\u5316\u914D\u7F6Eexample # \u4F7F\u7528kafka\u4E0A\u62A5\u7684\u6A21\u5F0F datarangers.sdk.mode=kafka datarangers.sdk.kafka.bootstrapServers={ip1}:9192,{ip2}:9192 diff --git a/datarangers-sdk-example/src/main/resources/application-saas.properties b/datarangers-sdk-example/src/main/resources/application-saas.properties index c9f7af0..1ea80d2 100644 --- a/datarangers-sdk-example/src/main/resources/application-saas.properties +++ b/datarangers-sdk-example/src/main/resources/application-saas.properties @@ -1,13 +1,23 @@ +# saas \u7F6Eexample datarangers.sdk.env=saas datarangers.sdk.mode=http +# [domain] # \u670D\u52A1\u5668ip\u6216\u57DF\u540D +# \u56FD\u5185 datarangers.sdk.domain=https://mcs.ctobsnssdk.com -# app key +# \u56FD\u9645 +#datarangers.sdk.domain=https://mcs.tobsnssdk.com + +# [app key] datarangers.sdk.appKeys.${SDK_APP_1}=${SDK_APP_KEY_1} +# [openapi] # openapi\u7684domain +# \u56FD\u5185 datarangers.sdk.openapiConfig.domain=https://analytics.volcengineapi.com +# \u56FD\u9645 +#datarangers.sdk.openapiConfig.domain=https://analytics.byteplusapi.com # openapi\u7684ak, sk datarangers.sdk.openapiConfig.ak=${OPENAPI_AK} From 5de67dab8b28457a89f5d5f3708cdc85e4fc979c Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Mon, 10 Oct 2022 14:56:23 +0800 Subject: [PATCH 15/21] =?UTF-8?q?=E8=B0=83=E6=95=B4pom=E5=B7=A5=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- datarangers-sdk-core/pom.xml | 3 +- datarangers-sdk-example/pom.xml | 36 +++++++++++++++------ datarangers-sdk-starter/pom.xml | 12 +++---- pom.xml | 57 ++++++++++++++++++++++++++++++--- 4 files changed, 84 insertions(+), 24 deletions(-) diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml index 236a816..341f485 100644 --- a/datarangers-sdk-core/pom.xml +++ b/datarangers-sdk-core/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - 1.5.6-SNAPSHOT + t1-1.5.6-release 4.0.0 jar @@ -28,7 +28,6 @@ com.datarangers datarangers-sdk-core - 1.5.6-SNAPSHOT diff --git a/datarangers-sdk-example/pom.xml b/datarangers-sdk-example/pom.xml index debd429..86ea7ca 100644 --- a/datarangers-sdk-example/pom.xml +++ b/datarangers-sdk-example/pom.xml @@ -3,14 +3,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.boot - spring-boot-starter-parent - 2.7.4 - + datarangers-sdk + com.datarangers + t1-1.5.6-release com.datarangers datarangers-sdk-example - 0.0.1-SNAPSHOT datarangers-sdk-example datarangers-sdk-example @@ -20,7 +18,7 @@ com.datarangers datarangers-sdk-starter - 1.5.6-snapshot + ${project.version} org.springframework.boot @@ -35,8 +33,7 @@ com.datarangers datarangers-sdk-core - 1.5.6-SNAPSHOT - compile + ${project.version} junit @@ -49,8 +46,27 @@ - org.springframework.boot - spring-boot-maven-plugin + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + UTF-8 + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + diff --git a/datarangers-sdk-starter/pom.xml b/datarangers-sdk-starter/pom.xml index 1bcf627..77cd046 100644 --- a/datarangers-sdk-starter/pom.xml +++ b/datarangers-sdk-starter/pom.xml @@ -3,9 +3,12 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 1.5.6-SNAPSHOT + + com.datarangers + datarangers-sdk + t1-1.5.6-release + jar - com.datarangers datarangers-sdk-starter datarangers-server-sdk-starter A SDK for datarangers user @@ -27,17 +30,14 @@ org.springframework.boot spring-boot-autoconfigure - 2.3.4.RELEASE org.apache.httpcomponents.client5 httpclient5 - 5.0.1 org.springframework.boot spring-boot-starter-test - 2.3.4.RELEASE test @@ -49,13 +49,11 @@ org.springframework.boot spring-boot-test - 2.3.4.RELEASE test log4j-to-slf4j org.apache.logging.log4j - 2.15.0 test diff --git a/pom.xml b/pom.xml index eee4483..240822c 100644 --- a/pom.xml +++ b/pom.xml @@ -3,18 +3,18 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - 1.5.6-SNAPSHOT - - com.datarangers datarangers-sdk pom - 1.5.6-SNAPSHOT + t1-1.5.6-release datarangers-sdk-core datarangers-sdk-starter + datarangers-sdk-example + + 2.3.4.RELEASE + @@ -24,6 +24,53 @@ maven-source-plugin 3.2.1 + + com.datarangers + datarangers-sdk-core + ${project.version} + + + org.springframework.boot + spring-boot-autoconfigure + 2.3.4.RELEASE + + + org.apache.httpcomponents.client5 + httpclient5 + 5.0.1 + + + org.springframework.boot + spring-boot-starter + ${spring.boot.version} + + + log4j-to-slf4j + org.apache.logging.log4j + + + + + org.springframework.boot + spring-boot-starter-test + ${spring.boot.version} + + + log4j-to-slf4j + org.apache.logging.log4j + + + + + org.springframework.boot + spring-boot-test + ${spring.boot.version} + + + log4j-to-slf4j + org.apache.logging.log4j + 2.15.0 + From c65d17393464f4674526d538daf6b15db6be7a01 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Thu, 20 Oct 2022 10:10:43 +0800 Subject: [PATCH 16/21] fix: version --- README.md | 2 +- .../src/main/java/com/datarangers/config/Constants.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0efc3bf..e598796 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ datarangers-sdk-java 是 [DataFinder](https://www.volcengine.com/product/datafin com.datarangers datarangers-sdk-core - 1.5.5-release + t1-1.5.6-release ``` diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java index add87dc..9566e7a 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java @@ -11,7 +11,7 @@ import java.util.GregorianCalendar; public class Constants { - public static final String SDK_VERSION = "datarangers_sdk_1.5.5-release"; + public static final String SDK_VERSION = "datarangers_sdk_t1-1.5.6-release"; public static DateTimeFormatter FULL_HOUR = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH"); public static DateTimeFormatter FULL_DAY = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); public static final String APP_LOG_PATH = "/sdk/log"; From 8b8f4ccca9cb18889e919985d4b114f0dd495245 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Thu, 20 Oct 2022 10:28:49 +0800 Subject: [PATCH 17/21] fix: remove dup log --- .../src/main/java/com/datarangers/collector/Collector.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java index 6d100eb..af4a1ab 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java @@ -52,8 +52,6 @@ public abstract class Collector implements EventCollector { private static volatile Boolean IS_INIT = false; public Collector(String appType, DataRangersSDKConfigProperties properties, Callback cb) { - logger.info("sdk config properties: \r\n{}", properties); - System.out.println("sdk config properties: \r\n" + properties.toString()); this.appType = appType; this.enable = properties.isEnable(); this.properties = properties; @@ -213,8 +211,8 @@ private void init() { initConsumer(); initHook(); IS_INIT = true; - System.out.println(String.format("sdk config: %s\r\n", this.properties)); - logger.info("sdk config: %s\r\n", this.properties); + logger.info("sdk config properties: \r\n{}", properties); + System.out.println("sdk config properties: \r\n" + properties); } } } From 7cc37f0c231815af3d9a2d6ae21332d1b63fa420 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Thu, 20 Oct 2022 10:31:43 +0800 Subject: [PATCH 18/21] fix: update version --- README.md | 2 +- datarangers-sdk-core/pom.xml | 2 +- .../src/main/java/com/datarangers/config/Constants.java | 2 +- datarangers-sdk-example/pom.xml | 2 +- datarangers-sdk-starter/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e598796..9d7fc0f 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ datarangers-sdk-java 是 [DataFinder](https://www.volcengine.com/product/datafin com.datarangers datarangers-sdk-core - t1-1.5.6-release + t2-1.5.6-release ``` diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml index 341f485..cf6c524 100644 --- a/datarangers-sdk-core/pom.xml +++ b/datarangers-sdk-core/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - t1-1.5.6-release + t2-1.5.6-release 4.0.0 jar diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java index 9566e7a..0172926 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java @@ -11,7 +11,7 @@ import java.util.GregorianCalendar; public class Constants { - public static final String SDK_VERSION = "datarangers_sdk_t1-1.5.6-release"; + public static final String SDK_VERSION = "datarangers_sdk_t2-1.5.6-release"; public static DateTimeFormatter FULL_HOUR = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH"); public static DateTimeFormatter FULL_DAY = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); public static final String APP_LOG_PATH = "/sdk/log"; diff --git a/datarangers-sdk-example/pom.xml b/datarangers-sdk-example/pom.xml index 86ea7ca..2fef3f5 100644 --- a/datarangers-sdk-example/pom.xml +++ b/datarangers-sdk-example/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - t1-1.5.6-release + t2-1.5.6-release com.datarangers datarangers-sdk-example diff --git a/datarangers-sdk-starter/pom.xml b/datarangers-sdk-starter/pom.xml index 77cd046..0bab138 100644 --- a/datarangers-sdk-starter/pom.xml +++ b/datarangers-sdk-starter/pom.xml @@ -6,7 +6,7 @@ com.datarangers datarangers-sdk - t1-1.5.6-release + t2-1.5.6-release jar datarangers-sdk-starter diff --git a/pom.xml b/pom.xml index 240822c..4bcb8b2 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.datarangers datarangers-sdk pom - t1-1.5.6-release + t2-1.5.6-release datarangers-sdk-core datarangers-sdk-starter From 8c35ee8916b8188bb7cea7f6e7cb4d86f710d2d8 Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Tue, 1 Nov 2022 14:47:04 +0800 Subject: [PATCH 19/21] fix: fix kafka error & change version --- README.md | 2 +- datarangers-sdk-core/pom.xml | 2 +- .../src/main/java/com/datarangers/collector/Collector.java | 2 +- .../src/main/java/com/datarangers/config/Constants.java | 2 +- datarangers-sdk-example/pom.xml | 2 +- .../main/java/com/datarangers/example/KafkaPriSdkExample.java | 2 +- datarangers-sdk-starter/pom.xml | 2 +- pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 9d7fc0f..67b5736 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ datarangers-sdk-java 是 [DataFinder](https://www.volcengine.com/product/datafin com.datarangers datarangers-sdk-core - t2-1.5.6-release + t3-1.5.6-release ``` diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml index cf6c524..f58ed68 100644 --- a/datarangers-sdk-core/pom.xml +++ b/datarangers-sdk-core/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - t2-1.5.6-release + t3-1.5.6-release 4.0.0 jar diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java index af4a1ab..b6649e2 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/collector/Collector.java @@ -48,7 +48,7 @@ public abstract class Collector implements EventCollector { protected DataRangersSDKConfigProperties properties; protected Callback callback; protected Consumer consumer = null; - protected KafkaProducer kafkaProducer; + protected static KafkaProducer kafkaProducer; private static volatile Boolean IS_INIT = false; public Collector(String appType, DataRangersSDKConfigProperties properties, Callback cb) { diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java index 0172926..1c9d058 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java @@ -11,7 +11,7 @@ import java.util.GregorianCalendar; public class Constants { - public static final String SDK_VERSION = "datarangers_sdk_t2-1.5.6-release"; + public static final String SDK_VERSION = "datarangers_sdk_t3-1.5.6-release"; public static DateTimeFormatter FULL_HOUR = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH"); public static DateTimeFormatter FULL_DAY = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); public static final String APP_LOG_PATH = "/sdk/log"; diff --git a/datarangers-sdk-example/pom.xml b/datarangers-sdk-example/pom.xml index 2fef3f5..62e9af3 100644 --- a/datarangers-sdk-example/pom.xml +++ b/datarangers-sdk-example/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - t2-1.5.6-release + t3-1.5.6-release com.datarangers datarangers-sdk-example diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java index e3f7827..a171959 100644 --- a/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/KafkaPriSdkExample.java @@ -49,9 +49,9 @@ public class KafkaPriSdkExample extends AbstractSdkExample { // kafkaConfig.setProperties(kafkaProperties); // 初始化collector - appEventCollector = new AppEventCollector("app", properties); webEventCollector = new AppEventCollector("web", properties); mpEventCollector = new AppEventCollector("mp", properties); + appEventCollector = new AppEventCollector("app", properties); } diff --git a/datarangers-sdk-starter/pom.xml b/datarangers-sdk-starter/pom.xml index 0bab138..c414450 100644 --- a/datarangers-sdk-starter/pom.xml +++ b/datarangers-sdk-starter/pom.xml @@ -6,7 +6,7 @@ com.datarangers datarangers-sdk - t2-1.5.6-release + t3-1.5.6-release jar datarangers-sdk-starter diff --git a/pom.xml b/pom.xml index 4bcb8b2..25528cb 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.datarangers datarangers-sdk pom - t2-1.5.6-release + t3-1.5.6-release datarangers-sdk-core datarangers-sdk-starter From 486912b46221f452561f2fe4fc302f45a7290b2d Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Mon, 7 Nov 2022 20:19:17 +0800 Subject: [PATCH 20/21] release: to 1.5.6 --- README.md | 2 +- datarangers-sdk-core/pom.xml | 2 +- .../src/main/java/com/datarangers/config/Constants.java | 2 +- datarangers-sdk-example/pom.xml | 2 +- datarangers-sdk-starter/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 67b5736..f03629e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ datarangers-sdk-java 是 [DataFinder](https://www.volcengine.com/product/datafin com.datarangers datarangers-sdk-core - t3-1.5.6-release + 1.5.6-release ``` diff --git a/datarangers-sdk-core/pom.xml b/datarangers-sdk-core/pom.xml index f58ed68..3d035e1 100644 --- a/datarangers-sdk-core/pom.xml +++ b/datarangers-sdk-core/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - t3-1.5.6-release + 1.5.6-release 4.0.0 jar diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java index 1c9d058..73e222c 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/config/Constants.java @@ -11,7 +11,7 @@ import java.util.GregorianCalendar; public class Constants { - public static final String SDK_VERSION = "datarangers_sdk_t3-1.5.6-release"; + public static final String SDK_VERSION = "datarangers_sdk_1.5.6-release"; public static DateTimeFormatter FULL_HOUR = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH"); public static DateTimeFormatter FULL_DAY = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); public static final String APP_LOG_PATH = "/sdk/log"; diff --git a/datarangers-sdk-example/pom.xml b/datarangers-sdk-example/pom.xml index 62e9af3..a529ff3 100644 --- a/datarangers-sdk-example/pom.xml +++ b/datarangers-sdk-example/pom.xml @@ -5,7 +5,7 @@ datarangers-sdk com.datarangers - t3-1.5.6-release + 1.5.6-release com.datarangers datarangers-sdk-example diff --git a/datarangers-sdk-starter/pom.xml b/datarangers-sdk-starter/pom.xml index c414450..98febdf 100644 --- a/datarangers-sdk-starter/pom.xml +++ b/datarangers-sdk-starter/pom.xml @@ -6,7 +6,7 @@ com.datarangers datarangers-sdk - t3-1.5.6-release + 1.5.6-release jar datarangers-sdk-starter diff --git a/pom.xml b/pom.xml index 25528cb..c4d291c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.datarangers datarangers-sdk pom - t3-1.5.6-release + 1.5.6-release datarangers-sdk-core datarangers-sdk-starter From ffaac530c4084207e74a4c54a3221e08d992e38a Mon Sep 17 00:00:00 2001 From: "zhangpeng.spin" Date: Wed, 30 Nov 2022 18:26:13 +0800 Subject: [PATCH 21/21] =?UTF-8?q?feature:=20profile=20=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=94=AF=E6=8C=81=20header?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../collector/AppEventCollector.java | 46 +++++++++++++- .../datarangers/collector/EventCollector.java | 25 ++++++++ .../example/HttpPriSdkProfileExample.java | 61 +++++++++++++++++++ 3 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkProfileExample.java diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/collector/AppEventCollector.java b/datarangers-sdk-core/src/main/java/com/datarangers/collector/AppEventCollector.java index 927d4d0..719d44a 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/collector/AppEventCollector.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/collector/AppEventCollector.java @@ -6,7 +6,6 @@ */ package com.datarangers.collector; -import com.datarangers.asynccollector.Consumer; import com.datarangers.config.Constants; import com.datarangers.config.DataRangersSDKConfigProperties; import com.datarangers.config.RangersJSONConfig; @@ -16,8 +15,8 @@ import com.datarangers.message.MessageType; import com.datarangers.profile.ItemMethod; import com.datarangers.profile.ProfileMethod; - import com.datarangers.sender.Callback; + import java.io.IOException; import java.util.*; @@ -104,6 +103,22 @@ private void profile(String userUniqueId, int appId, ProfileMethod method, sendEvent(header, event, MessageType.PROFILE); } + private void profile(Header header, ProfileMethod method, + Map profiles) { + if (header == null || header.getAppId() == null) { + logger.error("header is null or appId is empty"); + return; + } + if (profiles == null) { + logger.error("userUniqueId={}, app_id={}, device_id={} params are null.", + header.getUserUniqueId(), header.getAppId(), header.getDeviceId()); + return; + } + Event event = new EventV3().setEvent(method.toString()).setParams(profiles) + .setUserId(header.getUserUniqueId()); + sendEvent(header, event, MessageType.PROFILE); + } + @Override public void profileSet(String userUniqueId, int appId, Map eventParams) { profile(userUniqueId, appId, ProfileMethod.SET, eventParams); @@ -126,6 +141,33 @@ public void profileUnset(String userUniqueId, int appId, List params) { profile(userUniqueId, appId, ProfileMethod.UNSET, eventParams); } + @Override + public void profileSet(Header header, Map profiles) { + profile(header, ProfileMethod.SET, profiles); + } + + @Override + public void profileSetOnce(Header header, Map profiles) { + profile(header, ProfileMethod.SET_ONCE, profiles); + } + + @Override + public void profileIncrement(Header header, Map profiles) { + profile(header, ProfileMethod.INCREMENT, profiles); + } + + @Override + public void profileAppend(Header header, Map profiles) { + profile(header, ProfileMethod.APPEND, profiles); + } + + @Override + public void profileUnset(Header header, List profiles) { + Map eventParams = new HashMap<>(); + profiles.forEach(p -> eventParams.put(p, "java")); + profile(header, ProfileMethod.UNSET, eventParams); + } + @Override public void profileAppend(String userUniqueId, int appId, Map eventParams) { profile(userUniqueId, appId, ProfileMethod.APPEND, eventParams); diff --git a/datarangers-sdk-core/src/main/java/com/datarangers/collector/EventCollector.java b/datarangers-sdk-core/src/main/java/com/datarangers/collector/EventCollector.java index 3426ddb..135b1c2 100644 --- a/datarangers-sdk-core/src/main/java/com/datarangers/collector/EventCollector.java +++ b/datarangers-sdk-core/src/main/java/com/datarangers/collector/EventCollector.java @@ -115,6 +115,31 @@ public interface EventCollector { */ void profileUnset(String userUniqueId, int appId, List params); + /** + * 功能描述: 对用户进行profile属性设置 + * + * @param header 用户相关信息 + * @param profiles 需要设置的用户属性 + * @return: void + * @date 2020/12/23 10:43 + */ + void profileSet(Header header, Map profiles); + + void profileSetOnce(Header header, Map profiles); + + void profileIncrement(Header header, Map profiles); + + void profileAppend(Header header, Map profiles); + + /** + * 功能描述: 删除用户的属性 + * @param header 用户相关信息 + * @param profiles 需要删除的用户属性名 + * @return: void + * @date 2020/12/25 16:11 + */ + void profileUnset(Header header, List profiles); + /** * 功能描述: 对业务对象进行设置 * diff --git a/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkProfileExample.java b/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkProfileExample.java new file mode 100644 index 0000000..507c76c --- /dev/null +++ b/datarangers-sdk-example/src/main/java/com/datarangers/example/HttpPriSdkProfileExample.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 Beijing Volcano Engine Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.datarangers.example; + +import com.datarangers.collector.EventCollector; +import com.datarangers.event.Header; +import com.datarangers.event.HeaderV3; + +import java.util.HashMap; + +/** + * 在私有化场景，直接使用 Http 模式使用进行发送事件 + * + * @Author zhangpeng.spin@bytedance.com + * @Date 2022/9/28 + */ +public class HttpPriSdkProfileExample extends HttpPriSdkExample { + + HttpPriSdkProfileExample() { + super(); + } + + public void sendUserProfileWithHeader(Header header) { + EventCollector appEventCollector = getAppEventCollector(); + appEventCollector.profileSet(header, new HashMap() {{ + put("profile_a", "param_11"); + put("profile_b", "param_22"); + put("profile_c", "param_33"); + put("profile_d", "param_55"); + }}); + } + + public static void main(String[] args) { + HttpPriSdkProfileExample sdkExample = new HttpPriSdkProfileExample(); + String userUniqueId = "test_sdk_user1"; + Long deviceId = 123456L; + int appId = 10000001; + + // 上报用户属性 + Header header = new HeaderV3.Builder() + .setUserUniqueId(userUniqueId) + .setDeviceId(deviceId) + .setAppId(appId) + .build(); + + sdkExample.sendUserProfileWithHeader(header); + + // 指定localTimeMs时间,即事件发生时间 + long localTimeMs = System.currentTimeMillis(); + sdkExample.sendEventWithLocalTimeMs(userUniqueId, appId, localTimeMs); + } + +}