Skip to content

Commit 42b7c29

Browse files
leonelpenaleonelpena
committed
Support for Synchronous query and log5j added, and time out of the
cursor disabled
1 parent 90ecd4e commit 42b7c29

File tree

6 files changed

+262
-14
lines changed

6 files changed

+262
-14
lines changed

‎CurrentDesignToOneDocumentDesign/.classpath‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<classpath>
33
<classpathentry kind="src" path="src"/>
4+
<classpathentry kind="src" path="etc"/>
45
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
56
<classpathentry kind="lib" path="/opt/hypertable/0.9.6.5/lib/java/slf4j-log4j12-1.5.8.jar"/>
67
<classpathentry kind="lib" path="/opt/hypertable/0.9.6.5/lib/java/slf4j-api-1.5.8.jar"/>

‎CurrentDesignToOneDocumentDesign/etc/log4j.properties‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#log4j.rootCategory=ALL, Default
2+
#log4j.appender.Default=org.apache.log4j.FileAppender
3+
#log4j.appender.Default.file=../log/server.log
4+
#log4j.appender.Default.layout=org.apache.log4j.xml.XMLLayout
5+
#log4j.appender.Default.append=false
6+
7+
# This sets the global logging level and specifies the appenders
8+
log4j.rootLogger=DEBUG, myConsoleAppender, File
9+
#log4j.rootLogger=DEBUG, File
10+
11+
# settings for the console appender
12+
log4j.appender.myConsoleAppender=org.apache.log4j.ConsoleAppender
13+
log4j.appender.myConsoleAppender.layout=org.apache.log4j.PatternLayout
14+
log4j.appender.myConsoleAppender.layout.ConversionPattern=%-5p %c %x - %m%n
15+
16+
# settings for the file appender
17+
log4j.appender.File=org.apache.log4j.FileAppender
18+
log4j.appender.File.file=server.log
19+
log4j.appender.File.layout=org.apache.log4j.PatternLayout
20+
log4j.appender.File.append=false
21+
log4j.appender.FILE.layout.ConversionPattern=%d{yyyy-MM-dd}-%t-%x-%-5p-%-10c:%m%n

‎CurrentDesignToOneDocumentDesign/src/cl/alma/onedocument/Main.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ public static void main(String[] args) {
1818
try {
1919
//mongoManager = new MongoManager("localhost",
2020
mongoManager = new MongoManager("mongo-r1.osf.alma.cl",
21-
"OneDocumentPerComponentPerDay", "monitorData");
21+
//"OneDocumentPerComponentPerDay", "monitorData");
22+
"OneMonitorPointPerDayPerDocument", "monitorData");
2223
mongoManager.setQueue(queue);
2324
} catch (UnknownHostException e1) {
2425
e1.printStackTrace();
@@ -27,7 +28,7 @@ public static void main(String[] args) {
2728
Thread hiloMongo = new Thread(mongoManager);
2829
hiloMongo.start();
2930

30-
try {
31+
/*try {
3132
Query query = new Query("mongo-r1.osf.alma.cl", "MONDB",
3233
"monitorPoints");
3334
query.setQueue(queue);
@@ -47,7 +48,7 @@ public static void main(String[] args) {
4748
} catch (UnknownHostException e) {
4849
e.printStackTrace();
4950
System.exit(-1);
50-
}
51+
}*/
5152
}
5253

5354
}

‎CurrentDesignToOneDocumentDesign/src/cl/alma/onedocument/MongoManager.java‎

Lines changed: 166 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,32 @@
11
package cl.alma.onedocument;
22

33
import java.net.UnknownHostException;
4+
import java.util.Arrays;
45
import java.util.Calendar;
56
import java.util.Date;
67
import java.util.GregorianCalendar;
78
import java.util.HashMap;
89
import java.util.Map;
910
import java.util.concurrent.LinkedBlockingQueue;
1011

12+
import org.apache.log4j.Logger;
13+
1114
import com.mongodb.BasicDBObject;
1215
import com.mongodb.DB;
1316
import com.mongodb.DBCollection;
17+
import com.mongodb.DBCursor;
1418
import com.mongodb.DBObject;
1519
import com.mongodb.Mongo;
1620

1721
public class MongoManager implements Runnable {
22+
private static final Logger log = Logger.getLogger(MongoManager.class);
23+
1824
private Mongo mongo;
1925
private DB database;
20-
private HashMap<String, DBCollection> mongoCollections;
26+
//private HashMap<String, DBCollection> mongoCollections;
27+
private HashMap<Integer, DBCollection> mongoCollections;
28+
private DBCollection multipleCollection;
29+
2130
private DBCollection collection;
2231
private LinkedBlockingQueue<DBObject> queue;
2332

@@ -26,25 +35,44 @@ public MongoManager(String host, String dbname, String coll)
2635

2736
mongo = new Mongo(host);
2837
database = mongo.getDB(dbname);
29-
mongoCollections = new HashMap<String, DBCollection>(70);
38+
//mongoCollections = new HashMap<String, DBCollection>(70);
39+
mongoCollections = new HashMap<Integer, DBCollection>(70);
3040
//collection = database.getCollection("monitorData");
3141
collection = database.getCollection(coll);
3242
}
3343

3444
public void setQueue(LinkedBlockingQueue<DBObject> queue) {
3545
this.queue = queue;
3646
}
37-
47+
3848
public DBCollection getCollection(DocumentID id) {
3949
//String key = id.getAntenna() + DocumentID.SEPARATOR + id.getComponent();
40-
String key = id.toString();
50+
//String key = id.toString();
51+
52+
// Using the month as the key
53+
Integer key = id.getMonth();
4154

4255
if (mongoCollections.containsKey(key))
4356
return mongoCollections.get(key);
4457

58+
BasicDBObject index = null;
59+
if (!database.collectionExists("monitorData_"+key)) {
60+
System.out.println("la coleccion no existe");
61+
index = new BasicDBObject("metadata.date", 1);
62+
index.append("metadata.monitorPoint", 1);
63+
index.append("metadata.antenna", 1);
64+
index.append("metadata.component", 1);
65+
}
66+
4567
// if the collection does not exist, it will be created automatically
46-
DBCollection c = database.getCollection("monData_"+key);
68+
DBCollection c = database.getCollection("monitorData_"+key);
4769
mongoCollections.put(key, c);
70+
71+
if (index!=null) {
72+
//c.createIndex(index);
73+
c.ensureIndex(index);
74+
}
75+
4876
return c;
4977
}
5078

@@ -79,7 +107,11 @@ public void upsert(Metadata metadata, int hour, int minute,
79107
//System.out.println("Document: "+document);
80108
//System.out.println("Update Document: "+updateDocument);
81109

82-
collection.update(document, updateDocument, true, false);
110+
//collection.update(document, updateDocument, true, false);
111+
112+
// Codigo para usar colecciones mensuales.
113+
multipleCollection = getCollection(metadata.getDocumentID());
114+
multipleCollection.update(document, updateDocument, true, false);
83115
}
84116

85117
/*public void update(DocumentID id, int hour, int minute, int second,
@@ -135,9 +167,10 @@ public static void main(String[] args) {
135167
mongoManager.upsert(metadata, 14, 4, 6, "12345");
136168
}
137169

170+
/*
138171
@Override
139172
public void run() {
140-
int cont=0;
173+
int cont=0, error=0;
141174
try {
142175
while (true) {
143176
DBObject object = queue.take();
@@ -147,14 +180,18 @@ public void run() {
147180
//System.out.println("Set: "+mySet);
148181
149182
150-
/*if (myMap.get("date") instanceof Date) {
183+
//if (myMap.get("date") instanceof Date) {
151184
//System.out.println("Es instacia de DATE!!");
152-
} else {
185+
//} else {
153186
//System.out.println("NO lo es!");
154-
}*/
187+
//}
155188
156189
Calendar calendar = Calendar.getInstance();
157-
calendar.setTime((Date)myMap.get("date"));
190+
try {
191+
calendar.setTime((Date)myMap.get("date"));
192+
} catch (NullPointerException e) {
193+
System.err.println("Object: "+myMap);
194+
}
158195
159196
// ************************************************ //
160197
// Se añaden las tres horas de diferencia //
@@ -213,9 +250,127 @@ public void run() {
213250
}
214251
} catch (InterruptedException e) {
215252
253+
} catch (Throwable e) {
254+
error++;
255+
//System.err.println("Exception caught: "+e.getMessage());
256+
log.error("Exception caught: "+e.getMessage());
216257
} finally {
217258
close();
218259
System.out.println("Registros insertados: "+cont);
260+
System.out.println("Errores: "+error);
261+
}
262+
}*/
263+
264+
@Override
265+
public void run() {
266+
SynchronousQuery synQuery = null;
267+
try {
268+
synQuery = new SynchronousQuery("mongo-r1.osf.alma.cl", "MONDB",
269+
"monitorPoints");
270+
} catch (UnknownHostException e1) {
271+
System.exit(-1);
272+
}
273+
274+
DBCursor cursor = synQuery.exportData();
275+
276+
int cont=0, error=0;
277+
while (cursor.hasNext()) {
278+
try {
279+
//DBObject object = queue.take();
280+
DBObject object = cursor.next();
281+
282+
//Set<String> mySet = object.keySet();
283+
Map<String, Object> myMap = object.toMap();
284+
//System.out.println("Set: "+mySet);
285+
286+
287+
/*if (myMap.get("date") instanceof Date) {
288+
//System.out.println("Es instacia de DATE!!");
289+
} else {
290+
//System.out.println("NO lo es!");
291+
}*/
292+
293+
Calendar calendar = Calendar.getInstance();
294+
try {
295+
calendar.setTime((Date)myMap.get("date"));
296+
} catch (NullPointerException e) {
297+
log.error("NullPointerException: "+myMap);
298+
error++;
299+
}
300+
301+
// ************************************************ //
302+
// Se añaden las tres horas de diferencia //
303+
// con el servidor de mongo. //
304+
// ************************************************ //
305+
calendar.add(Calendar.HOUR, 3);
306+
307+
int year = calendar.get(Calendar.YEAR);
308+
int month = calendar.get(Calendar.MONTH)+1;
309+
int day = calendar.get(Calendar.DAY_OF_MONTH);
310+
311+
int hour = calendar.get(Calendar.HOUR_OF_DAY);
312+
int minute = calendar.get(Calendar.MINUTE);
313+
int second = calendar.get(Calendar.SECOND);
314+
315+
//System.out.println("Map: "+myMap+",\n year: "+year+", month: "+
316+
// month+", day: "+day+", hour: "+hour+", minute: "+minute+
317+
//", second: "+second);
318+
319+
// We need to split the componentName that comes from the old schema.
320+
// The format is "CONTROL/DV10/FrontEnd/Cryostat".
321+
// We extract the antenna, component and subcomponent
322+
// from it.
323+
String[] oldComponentName = ((String)myMap.get("componentName")).split("/");
324+
String antenna = oldComponentName[1];
325+
// Component and subcomponent are put together
326+
String component = null;
327+
328+
// If the old ComponentName length is three it means there is
329+
// just a component, however, if the length is four
330+
// there is a component and subcomponent
331+
if (oldComponentName.length==3) {
332+
component = oldComponentName[2];
333+
} else if (oldComponentName.length==4) {
334+
component = oldComponentName[2]+"/"+oldComponentName[3];
335+
} else {
336+
log.error("Error detected in component name: "+myMap.get("componentName"));
337+
}
338+
339+
//if (oldComponentName.length>3) {
340+
//component = oldComponentName[2]+"/"+oldComponentName[3];
341+
//} else {
342+
//component = oldComponentName[2];
343+
//}
344+
345+
String property = (String)myMap.get("propertyName");
346+
String monitorPoint = (String)myMap.get("monitorPointName");
347+
String location = (String)myMap.get("location");
348+
String serialNumber = (String)myMap.get("serialNumber");
349+
String monitorValue = (String)myMap.get("monitorValue");
350+
int index = Integer.parseInt(myMap.get("index").toString());
351+
int sampleTime = 0;
352+
353+
DocumentID documentID = new DocumentID(year, month, day,
354+
antenna, component, monitorPoint);
355+
//Metadata metadata = new Metadata(documentID, "ASDF Property", "TFING",
356+
// "as76d6fh", 5, 2);
357+
Metadata metadata = new Metadata(documentID, property,
358+
location, serialNumber, sampleTime, index);
359+
360+
//mongoManager.upsert(metadata, 14, 4, 6, "12345");
361+
upsert(metadata, hour, minute, second, monitorValue);
362+
cont++;
363+
//insert(record);
364+
} catch (Throwable e) {
365+
error++;
366+
log.error("Exception caught: "+e.getMessage());
367+
log.error(Arrays.toString(e.getStackTrace()));
368+
}
219369
}
370+
371+
synQuery.closeCursor();
372+
close();
373+
log.info("Registros insertados: "+cont);
374+
log.info("Errores: "+error);
220375
}
221376
}

‎CurrentDesignToOneDocumentDesign/src/cl/alma/onedocument/Query.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import org.bson.types.ObjectId;
99

1010
import com.mongodb.BasicDBObject;
11+
import com.mongodb.Bytes;
1112
import com.mongodb.DB;
1213
import com.mongodb.DBCollection;
1314
import com.mongodb.DBCursor;
@@ -45,6 +46,9 @@ public void exportData() {
4546
//query.append("componentName", "CONTROL/DV16/LLC");
4647
//query.append("monitorPointName", "POL_MON4");
4748

49+
// Used only when the query take more than 10 minutes.
50+
_collection.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
51+
4852
/* */
4953

5054
// Registros utilizados para probar la diferencia de la zona horaria

0 commit comments

Comments
 (0)