Skip to content

Commit ec03344

Browse files
liketicliketic
authored
Add resource api update (#186)
* Update list resource and pull logs sample * fix versio
1 parent 70a45b2 commit ec03344

File tree

6 files changed

+166
-89
lines changed

6 files changed

+166
-89
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
<groupId>com.aliyun.openservices</groupId>
1313
<artifactId>aliyun-log</artifactId>
14-
<version>0.6.130</version>
14+
<version>0.6.131</version>
1515
<packaging>jar</packaging>
1616

1717
<name>Aliyun LOG Java SDK</name>

‎src/main/java/com/aliyun/openservices/log/common/Consts.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ public String toString() {
9696
public static final String CONST_X_SLS_RAWDATASIZE = "x-log-rawdatasize";
9797
public static final String CONST_X_LOG_CURSOR_TIME = "x-log-cursor-time";
9898
public static final String CONST_X_LOG_END_OF_CURSOR = "x-log-end-of-cursor";
99+
public static final String CONST_X_READ_LAST_CURSOR = "x-log-read-last-cursor";
99100
public static final String CONST_X_SLS_SIGNATUREMETHOD = "x-log-signaturemethod";
100101
public static final String CONST_X_SLS_REQUESTID = "x-log-requestid";
101102
public static final String CONST_X_SLS_HOSTIP = "x-log-hostip";
@@ -413,6 +414,7 @@ public String toString() {
413414
public static final String RESOURCE_JSON_PATH_VALUE = "jsonPathValue";
414415
public static final String RESOURCE_REVERSE = "reverse";
415416
public static final String RESOURCE_SYSTEM_RECORDS = "includeSystemRecords";
417+
public static final String RESOURCE_JSON_FILTER_ACC = "jsonFilterAcc";
416418

417419
public static final String TOPOSTORE_URI = "/topostores";
418420
public static final String TOPOSTORE_OWNER = "owner";

‎src/main/java/com/aliyun/openservices/log/request/ListNextResourceRecordRequest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,17 @@ public class ListNextResourceRecordRequest extends RecordRequest {
1717
private String jsonPathValue;
1818
private Boolean reverse;
1919
private Boolean includeSystemRecords;
20+
private Boolean jsonFilterAcc = false;
2021
private List<String> recordIds = new ArrayList<String>();
2122

23+
public Boolean getJsonFilterAcc() {
24+
return jsonFilterAcc;
25+
}
26+
27+
public void setJsonFilterAcc(Boolean jsonFilterAcc) {
28+
this.jsonFilterAcc = jsonFilterAcc;
29+
}
30+
2231
public String getNextToken() {
2332
return nextToken;
2433
}
@@ -163,6 +172,10 @@ public Map<String, String> GetAllParams() {
163172
SetParam(Consts.RESOURCE_SYSTEM_RECORDS, "true");
164173
}
165174

175+
if (jsonFilterAcc) {
176+
SetParam(Consts.RESOURCE_JSON_FILTER_ACC, "true");
177+
}
178+
166179
if (recordIds != null && !recordIds.isEmpty()) {
167180
SetParam(Consts.RESOURCE_RECORD_IDS, Utils.join(",", recordIds));
168181
}

‎src/main/java/com/aliyun/openservices/log/request/ListResourceRecordRequest.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import java.util.ArrayList;
77
import java.util.List;
88
import java.util.Map;
9+
import java.util.Set;
910

1011
public class ListResourceRecordRequest extends RecordRequest {
1112
private Integer offset;
@@ -16,8 +17,17 @@ public class ListResourceRecordRequest extends RecordRequest {
1617
private String jsonPath;
1718
private String jsonPathValue;
1819
private Boolean includeSystemRecords;
20+
private Boolean jsonFilterAcc = false;
1921
private List<String> recordIds = new ArrayList<String>();
2022

23+
public Boolean getJsonFilterAcc() {
24+
return jsonFilterAcc;
25+
}
26+
27+
public void setJsonFilterAcc(Boolean jsonFilterAcc) {
28+
this.jsonFilterAcc = jsonFilterAcc;
29+
}
30+
2131
public Boolean getIncludeSystemRecords() {
2232
return includeSystemRecords;
2333
}
@@ -144,6 +154,10 @@ public Map<String, String> GetAllParams() {
144154
SetParam(Consts.RESOURCE_SYSTEM_RECORDS, "true");
145155
}
146156

157+
if (jsonFilterAcc) {
158+
SetParam(Consts.RESOURCE_JSON_FILTER_ACC, "true");
159+
}
160+
147161
return super.GetAllParams();
148162
}
149163

‎src/main/java/com/aliyun/openservices/log/response/PullLogsResponse.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ public class PullLogsResponse extends Response {
2424
private final String compressType;
2525
private final long cursorTime;
2626
private final boolean isEndOfCursor;
27+
private final String readLastCursor;
2728

2829
/**
2930
* Construct the response with http headers
@@ -49,6 +50,7 @@ public PullLogsResponse(Map<String, String> headers, byte[] rawData) throws LogE
4950
cursorTime = cursorTimeHeader != null && !cursorTimeHeader.isEmpty()
5051
? Long.parseLong(cursorTimeHeader.trim()) : 0;
5152
isEndOfCursor = "1".equals(headers.get(Consts.CONST_X_LOG_END_OF_CURSOR));
53+
readLastCursor = headers.get(Consts.CONST_X_READ_LAST_CURSOR);
5254
} catch (NumberFormatException e) {
5355
throw new LogException("ParseLogGroupListRawSizeError", e.getMessage(), e, GetRequestId());
5456
}
@@ -204,4 +206,7 @@ public byte[] getRawData() {
204206
return rawData;
205207
}
206208

209+
public String getReadLastCursor() {
210+
return readLastCursor;
211+
}
207212
}

‎src/main/java/com/aliyun/openservices/log/sample/PullLogsSample.java‎

Lines changed: 131 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -9,96 +9,139 @@
99
import com.aliyun.openservices.log.response.ListShardResponse;
1010
import com.aliyun.openservices.log.response.PullLogsResponse;
1111

12+
import java.nio.charset.StandardCharsets;
13+
import java.util.Base64;
1214
import java.util.List;
1315
import java.util.stream.Collectors;
1416

1517
class PullLogsSample {
16-
private final String endPoint = "";
17-
private final String akId = "your_access_id";
18-
private final String ak = "your_access_key";
19-
private final Client client = new Client(endPoint, akId, ak);
20-
private final String project = "your_project_name";
21-
private final String logStore = "your_log_store";
22-
23-
public static void main(String[] args) {
24-
// ------------------------Data API------------------------
25-
PullLogsSample sample = new PullLogsSample();
26-
// ------------------------Shard------------------------
27-
List<Integer> shardIds = sample.listShard();
28-
29-
if (shardIds != null && shardIds.size() > 0) {
30-
// choose a shard to getCursor and pullLogs
31-
Integer shardId = shardIds.get(0);
32-
sample.getCursor(shardId);
33-
sample.pullLogs(shardId);
34-
}
35-
36-
}
37-
38-
public void getCursor(int shardId) {
39-
GetCursorResponse res;
40-
try {
41-
long fromTime = (int) (System.currentTimeMillis() / 1000.0 - 3600);
42-
res = client.GetCursor(project, logStore, shardId, fromTime);
43-
System.out.println("Cursor:" + res.GetCursor());
44-
45-
res = client.GetCursor(project, logStore, shardId, CursorMode.BEGIN);
46-
System.out.println("Cursor:" + res.GetCursor());
47-
48-
res = client.GetCursor(project, logStore, shardId, CursorMode.END);
49-
System.out.println("shard_id:" + shardId + " Cursor:" + res.GetCursor());
50-
} catch (LogException e) {
51-
e.printStackTrace();
52-
}
53-
}
54-
55-
public void pullLogs(int shardId) {
56-
try {
57-
GetCursorResponse cursorRes = client.GetCursor(project,
58-
logStore, shardId, CursorMode.BEGIN);
59-
String cursor = cursorRes.GetCursor();
60-
int iteration = 100;
61-
62-
for (int i = 0; i < iteration; i++) {
63-
PullLogsRequest request = new PullLogsRequest(project, logStore, shardId, 1000, cursor);
64-
PullLogsResponse response = client.pullLogs(request);
65-
66-
String next_cursor = response.getNextCursor();
67-
System.out.print("The Next cursor:" + next_cursor);
68-
69-
List<LogGroupData> logGroups = response.getLogGroups();
70-
for (LogGroupData logGroup : logGroups) {
71-
FastLogGroup fastLogGroup = logGroup.GetFastLogGroup();
72-
System.out.println("Source:" + fastLogGroup.getSource());
73-
System.out.println("Topic:" + fastLogGroup.getTopic());
74-
for (FastLog log : fastLogGroup.getLogs()) {
75-
System.out.println("LogTime:" + log.getTime());
76-
List<FastLogContent> contents = log.getContents();
77-
for (FastLogContent content : contents) {
78-
System.out.println(content.getKey() + ":" + content.getValue());
79-
}
80-
}
81-
}
82-
83-
if (cursor.equals(next_cursor)) {
84-
break;
85-
}
86-
cursor = next_cursor;
87-
}
88-
89-
} catch (LogException e) {
90-
e.printStackTrace();
91-
}
92-
}
93-
94-
public List<Integer> listShard() {
95-
try {
96-
ListShardResponse res = client.ListShard(project, logStore);
97-
System.out.println("RequestId:" + res.GetRequestId());
98-
return res.GetShards().stream().map(Shard::getShardId).collect(Collectors.toList());
99-
} catch (LogException e) {
100-
e.printStackTrace();
101-
return null;
102-
}
103-
}
18+
private final String endPoint = "";
19+
private final String akId = "your_access_id";
20+
private final String ak = "your_access_key";
21+
private final Client client = new Client(endPoint, akId, ak);
22+
private final String project = "your_project_name";
23+
private final String logStore = "your_log_store";
24+
25+
public static void main(String[] args) {
26+
// ------------------------Data API------------------------
27+
PullLogsSample sample = new PullLogsSample();
28+
// ------------------------Shard------------------------
29+
List<Integer> shardIds = sample.listShard();
30+
31+
if (shardIds != null && shardIds.size() > 0) {
32+
// choose a shard to getCursor and pullLogs
33+
Integer shardId = shardIds.get(0);
34+
sample.getCursor(shardId);
35+
sample.pullLogs(shardId);
36+
sample.pullLogsWithID(shardId);
37+
}
38+
}
39+
40+
public void getCursor(int shardId) {
41+
GetCursorResponse res;
42+
try {
43+
long fromTime = (int) (System.currentTimeMillis() / 1000.0 - 3600);
44+
res = client.GetCursor(project, logStore, shardId, fromTime);
45+
System.out.println("Cursor:" + res.GetCursor());
46+
47+
res = client.GetCursor(project, logStore, shardId, CursorMode.BEGIN);
48+
System.out.println("Cursor:" + res.GetCursor());
49+
50+
res = client.GetCursor(project, logStore, shardId, CursorMode.END);
51+
System.out.println("shard_id:" + shardId + " Cursor:" + res.GetCursor());
52+
} catch (LogException e) {
53+
e.printStackTrace();
54+
}
55+
}
56+
57+
public void pullLogs(int shardId) {
58+
try {
59+
GetCursorResponse cursorRes = client.GetCursor(project,
60+
logStore, shardId, CursorMode.BEGIN);
61+
String cursor = cursorRes.GetCursor();
62+
int iteration = 100;
63+
64+
for (int i = 0; i < iteration; i++) {
65+
PullLogsRequest request = new PullLogsRequest(project, logStore, shardId, 1000, cursor);
66+
PullLogsResponse response = client.pullLogs(request);
67+
String next_cursor = response.getNextCursor();
68+
System.out.print("The Next cursor:" + next_cursor);
69+
70+
List<LogGroupData> logGroups = response.getLogGroups();
71+
for (LogGroupData logGroup : logGroups) {
72+
FastLogGroup fastLogGroup = logGroup.getFastLogGroup();
73+
System.out.println("Source:" + fastLogGroup.getSource());
74+
System.out.println("Topic:" + fastLogGroup.getTopic());
75+
for (FastLog log : fastLogGroup.getLogs()) {
76+
System.out.println("LogTime:" + log.getTime());
77+
List<FastLogContent> contents = log.getContents();
78+
for (FastLogContent content : contents) {
79+
System.out.println(content.getKey() + ":" + content.getValue());
80+
}
81+
}
82+
}
83+
84+
if (cursor.equals(next_cursor)) {
85+
break;
86+
}
87+
cursor = next_cursor;
88+
}
89+
90+
} catch (LogException e) {
91+
e.printStackTrace();
92+
}
93+
}
94+
95+
public List<Integer> listShard() {
96+
try {
97+
ListShardResponse res = client.ListShard(project, logStore);
98+
System.out.println("RequestId:" + res.GetRequestId());
99+
return res.GetShards().stream().map(Shard::getShardId).collect(Collectors.toList());
100+
} catch (LogException e) {
101+
e.printStackTrace();
102+
return null;
103+
}
104+
}
105+
106+
private static String encodeCursor(long offset) {
107+
byte[] cursorAsBytes = Base64.getEncoder().encode(String.valueOf(offset).getBytes(StandardCharsets.UTF_8));
108+
return new String(cursorAsBytes, StandardCharsets.UTF_8);
109+
}
110+
111+
public void pullLogsWithID(int shardId) {
112+
try {
113+
GetCursorResponse cursorRes = client.GetCursor(project,
114+
logStore, shardId, CursorMode.BEGIN);
115+
String cursor = cursorRes.GetCursor();
116+
PullLogsRequest request = new PullLogsRequest(project, logStore, shardId, 1000, cursor);
117+
PullLogsResponse response = client.pullLogs(request);
118+
119+
String next_cursor = response.getNextCursor();
120+
System.out.print("The Next cursor:" + next_cursor);
121+
List<LogGroupData> logGroups = response.getLogGroups();
122+
if (logGroups.isEmpty()) {
123+
return;
124+
}
125+
long offset = Long.parseLong(response.getReadLastCursor()) - logGroups.size() + 1;
126+
String lastCursor = encodeCursor(offset);
127+
String seqNoPrefix = shardId + "|";
128+
for (LogGroupData logGroup : logGroups) {
129+
FastLogGroup fastLogGroup = logGroup.getFastLogGroup();
130+
List<FastLog> logs = fastLogGroup.getLogs();
131+
for (int lIdx = 0; lIdx < logs.size(); lIdx++) {
132+
FastLog log = logs.get(lIdx);
133+
System.out.println("LogTime:" + log.getTime());
134+
List<FastLogContent> contents = log.getContents();
135+
for (FastLogContent content : contents) {
136+
System.out.println(content.getKey() + ":" + content.getValue());
137+
}
138+
String logId = seqNoPrefix + lastCursor + "|" + lIdx;
139+
System.out.println("LogId:" + logId);
140+
}
141+
lastCursor = encodeCursor(++offset);
142+
}
143+
} catch (LogException e) {
144+
e.printStackTrace();
145+
}
146+
}
104147
}

0 commit comments

Comments
 (0)