Skip to content

Commit 9f1c66b

Browse files
ruanshudongruanshudong
committed
stat & property delele old history db data
1 parent e27d830 commit 9f1c66b

File tree

12 files changed

+151
-80
lines changed

12 files changed

+151
-80
lines changed

‎AdminRegistryServer/DbProxy.cpp‎

Lines changed: 0 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,8 @@ vector<tars::TC_Mysql*> DbProxy::_mysqlReg;
3333
vector<TC_ThreadMutex*> DbProxy::_mysqlLocks;
3434
static std::atomic<int> g_index(0);
3535

36-
TC_Timer DbProxy::_timer;
3736
//保留历史发布包
3837
int DbProxy::_patchHistory = 200;
39-
//statdb表保留天数
40-
int DbProxy::_statReserveDay = 31;
41-
//property表保留天数
42-
int DbProxy::_propertyReserveDay = 31;
4338

4439
#define MYSQL_LOCK size_t nowIndex = ++g_index; TC_LockT<TC_ThreadMutex> Lock(*(_mysqlLocks[nowIndex % _mysqlLocks.size()]));
4540
#define MYSQL_INDEX _mysqlReg[nowIndex % _mysqlReg.size()]
@@ -71,21 +66,6 @@ int DbProxy::init(TC_Config *pconf)
7166
_patchHistory = 10;
7267
}
7368

74-
_statReserveDay = TC_Common::strto<int>(pconf->get("/tars/db_reserve<stat_reserve_time>", "31"));
75-
if(_statReserveDay < 7)
76-
{
77-
_statReserveDay = 7;
78-
}
79-
80-
_propertyReserveDay = TC_Common::strto<int>(pconf->get("/tars/db_reserve<property_reserve_time>", "31"));
81-
if(_propertyReserveDay < 7)
82-
{
83-
_propertyReserveDay = 7;
84-
}
85-
86-
_timer.startTimer();
87-
_timer.postCron("0 0 3 * * *", std::bind(&DbProxy::doReserveDb, this, "statdb", pconf));
88-
_timer.postCron("0 1 3 * * *", std::bind(&DbProxy::doReserveDb, this, "propertydb", pconf));
8969
}
9070
catch (TC_Config_Exception& ex)
9171
{
@@ -99,48 +79,6 @@ int DbProxy::init(TC_Config *pconf)
9979
return 0;
10080
}
10181

102-
void DbProxy::doReserveDb(const string path, TC_Config *pconf)
103-
{
104-
try
105-
{
106-
107-
vector<string> dbs = pconf->getDomainVector("/tars/" + path);
108-
109-
for(auto &db : dbs)
110-
{
111-
TC_DBConf tcDBConf;
112-
tcDBConf.loadFromMap(pconf->getDomainMap("/tars/" + path + "/" + db));
113-
114-
TC_Mysql mysql(tcDBConf);
115-
116-
string sql = "select table_name from information_schema.tables where table_schema='" + tcDBConf._database + "' and table_type='base table' order by table_name";
117-
118-
string suffix = TC_Common::tm2str(TNOW - _statReserveDay * 24 * 60 * 60, "%Y%m%d");
119-
120-
string tableName = pconf->get("/tars/statdb/" + db + "<tbname>") + suffix;
121-
122-
auto datas = mysql.queryRecord(sql);
123-
124-
for(size_t i = 0; i < datas.size(); i++)
125-
{
126-
string name = datas[i]["table_name"];
127-
128-
if(name < tableName)
129-
{
130-
mysql.execute("drop table " + name);
131-
}
132-
else
133-
{
134-
break;
135-
}
136-
}
137-
}
138-
139-
}catch(exception &ex)
140-
{
141-
TLOGERROR(__FUNCTION__ << " exception: " << ex.what() << endl);
142-
}
143-
}
14482

14583
int DbProxy::addTaskReq(const TaskReq &taskReq)
14684
{

‎AdminRegistryServer/DbProxy.h‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
#include "util/tc_mysql.h"
2424
#include "util/tc_file.h"
2525
#include "util/tc_singleton.h"
26-
#include "util/tc_timer.h"
2726
#include "Node.h"
2827
#include "servant/RemoteLogger.h"
2928
#include "AdminReg.h"
@@ -306,15 +305,6 @@ class DbProxy : public tars::TC_Singleton<DbProxy>
306305

307306
//保留历史发布记录
308307
static int _patchHistory;
309-
310-
//定时器
311-
static TC_Timer _timer;
312-
313-
//statdb表保留天数
314-
static int _statReserveDay;
315-
316-
//property表保留天数
317-
static int _propertyReserveDay;
318308
};
319309

320310
#define DBPROXY DbProxy::getInstance()

‎PropertyServer/PropertyServer.cpp‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,12 @@ void PropertyServer::initialize()
3535
_insertInterval = 5;
3636
}
3737

38+
_reserveDay = TC_Common::strto<int>(g_pconf->get("/tars/db_reserve<property_reserve_time>", "31"));
39+
if(_reserveDay < 2)
40+
{
41+
_reserveDay = 2;
42+
}
43+
3844
//获取业务线程个数
3945
string sAdapter = ServerConfig::Application + "." + ServerConfig::ServerName + ".PropertyObjAdapter";
4046
string sHandleNum = "/tars/application/server/";
@@ -77,6 +83,10 @@ void PropertyServer::initialize()
7783

7884
_reapThread->start();
7985

86+
_timer.startTimer();
87+
// _timer.postRepeated(10000, false, std::bind(&PropertyServer::doReserveDb, this, "propertydb", g_pconf));
88+
_timer.postCron("0 0 3 * * *", std::bind(&PropertyServer::doReserveDb, this, "propertydb", g_pconf));
89+
8090
TARS_ADD_ADMIN_CMD_PREFIX("tars.tarsproperty.randorder", PropertyServer::cmdSetRandOrder);
8191
}
8292
catch ( exception& ex )
@@ -90,6 +100,55 @@ void PropertyServer::initialize()
90100
exit( 0 );
91101
}
92102
}
103+
104+
105+
void PropertyServer::doReserveDb(const string path, TC_Config *pconf)
106+
{
107+
try
108+
{
109+
110+
vector<string> dbs = pconf->getDomainVector("/tars/" + path);
111+
112+
for(auto &db : dbs)
113+
{
114+
TC_DBConf tcDBConf;
115+
tcDBConf.loadFromMap(pconf->getDomainMap("/tars/" + path + "/" + db));
116+
117+
TC_Mysql mysql(tcDBConf);
118+
119+
string sql = "select table_name from information_schema.tables where table_schema='" + tcDBConf._database + "' and table_type='base table' order by table_name";
120+
121+
string suffix = TC_Common::tm2str(TNOW - _reserveDay * 24 * 60 * 60, "%Y%m%d") + "00";
122+
123+
string tableName = pconf->get("/tars/" + path + "/" + db + "<tbname>") + suffix;
124+
125+
// TLOGDEBUG(__FUNCTION__ << " tableName: " << tableName<< endl);
126+
127+
auto datas = mysql.queryRecord(sql);
128+
129+
for(size_t i = 0; i < datas.size(); i++)
130+
{
131+
string name = datas[i]["table_name"];
132+
133+
if(name < tableName)
134+
{
135+
TLOGDEBUG(__FUNCTION__ << ", drop name:" << name << ", tableName: " << tableName << ", " << (name < tableName) << endl);
136+
137+
mysql.execute("drop table " + name);
138+
}
139+
else
140+
{
141+
break;
142+
}
143+
}
144+
}
145+
146+
}catch(exception &ex)
147+
{
148+
TLOGERROR(__FUNCTION__ << " exception: " << ex.what() << endl);
149+
}
150+
}
151+
93152
string PropertyServer::getRandOrder(void)
94153
{
95154
return _randOrder;

‎PropertyServer/PropertyServer.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
#ifndef __PROPERTY_SERVER_H_
1818
#define __PROPERTY_SERVER_H_
19+
#include "util/tc_timer.h"
1920

2021
#include "servant/Application.h"
2122
#include "PropertyReapThread.h"
@@ -55,7 +56,9 @@ class PropertyServer : public Application
5556
PropertyHashMap * getHashMapBuff(int iIndex, int iBuffer) { return &(_hashmap[iIndex][iBuffer]); }
5657

5758
int getBuffNum() { return _buffNum; }
58-
59+
60+
void doReserveDb(const string path, TC_Config *pconf);
61+
5962
protected:
6063
/**
6164
* 初始化, 只会进程调用一次
@@ -92,6 +95,9 @@ class PropertyServer : public Application
9295

9396
int _buffNum;
9497

98+
int _reserveDay = 31;
99+
100+
TC_Timer _timer;
95101
};
96102

97103
extern PropertyServer g_app;

‎QueryPropertyServer/QueryServer.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
#include "util/tc_thread_mutex.h"
2424
#include "util/tc_lock.h"
2525
#include "util/tc_thread_pool.h"
26-
// #include "util/tc_atomic.h"
2726
#include "DbThread.h"
2827
#include "QueryDbThread.h"
2928

@@ -115,6 +114,7 @@ class QueryServer : public Application , public TC_ThreadMutex
115114
TC_ThreadPool _poolDb; //具体查询压缩维度后的数据库实例数据的线程池
116115

117116
set<string> _notTarsSlaveName; //匹配非tars被调服务名
117+
118118
};
119119

120120
extern QueryServer g_app;

‎StatServer/StatServer.cpp‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ void StatServer::initialize()
3434
_iInsertInterval = 5;
3535
}
3636

37+
_reserveDay = TC_Common::strto<int>(g_pconf->get("/tars/db_reserve<stat_reserve_time>", "31"));
38+
if(_reserveDay < 2)
39+
{
40+
_reserveDay = 2;
41+
}
42+
3743
//获取业务线程个数
3844
string sAdapter = ServerConfig::Application + "." + ServerConfig::ServerName + ".StatObjAdapter";
3945
string sHandleNum = "/tars/application/server/";
@@ -78,6 +84,10 @@ void StatServer::initialize()
7884
_pReapSSDThread = new ReapSSDThread();
7985
_pReapSSDThread->start();
8086

87+
_timer.startTimer();
88+
// _timer.postRepeated(10000, false, std::bind(&StatServer::doReserveDb, this, "statdb", g_pconf));
89+
_timer.postCron("0 0 3 * * *", std::bind(&StatServer::doReserveDb, this, "statdb", g_pconf));
90+
8191
TARS_ADD_ADMIN_CMD_PREFIX("tars.tarsstat.randorder", StatServer::cmdSetRandOrder);
8292
}
8393
catch ( exception& ex )
@@ -92,6 +102,53 @@ void StatServer::initialize()
92102
}
93103
}
94104

105+
void StatServer::doReserveDb(const string path, TC_Config *pconf)
106+
{
107+
try
108+
{
109+
110+
vector<string> dbs = pconf->getDomainVector("/tars/" + path);
111+
112+
for(auto &db : dbs)
113+
{
114+
TC_DBConf tcDBConf;
115+
tcDBConf.loadFromMap(pconf->getDomainMap("/tars/" + path + "/" + db));
116+
117+
TC_Mysql mysql(tcDBConf);
118+
119+
string sql = "select table_name from information_schema.tables where table_schema='" + tcDBConf._database + "' and table_type='base table' order by table_name";
120+
121+
string suffix = TC_Common::tm2str(TNOW - _reserveDay * 24 * 60 * 60, "%Y%m%d") + "00";
122+
123+
string tableName = pconf->get("/tars/" + path + "/" + db + "<tbname>") + suffix;
124+
125+
TLOGDEBUG(__FUNCTION__ << " tableName: " << tableName<< endl);
126+
127+
auto datas = mysql.queryRecord(sql);
128+
129+
for(size_t i = 0; i < datas.size(); i++)
130+
{
131+
string name = datas[i]["table_name"];
132+
133+
if(name < tableName)
134+
{
135+
TLOGDEBUG(__FUNCTION__ << ", drop name:" << name << ", tableName: " << tableName << ", " << (name < tableName) << endl);
136+
137+
mysql.execute("drop table " + name);
138+
}
139+
else
140+
{
141+
break;
142+
}
143+
}
144+
}
145+
146+
}catch(exception &ex)
147+
{
148+
TLOGERROR(__FUNCTION__ << " exception: " << ex.what() << endl);
149+
}
150+
}
151+
95152
map<string, string>& StatServer::getVirtualMasterIp(void)
96153
{
97154
return _mVirtualMasterIp;

‎StatServer/StatServer.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#ifndef __STAT_SERVER_H_
1818
#define __STAT_SERVER_H_
1919

20+
#include "util/tc_timer.h"
2021
#include "servant/Application.h"
2122
#include "servant/StatF.h"
2223
#include "StatHashMap.h"
@@ -66,6 +67,8 @@ class StatServer : public Application
6667

6768
int getBuffNum() { return _iBuffNum; }
6869

70+
void doReserveDb(const string path, TC_Config *pconf);
71+
6972
private:
7073
void initHashMap();
7174

@@ -93,6 +96,10 @@ class StatServer : public Application
9396
StatHashMap **_hashmap;
9497

9598
int _iBuffNum;
99+
100+
int _reserveDay = 31;
101+
102+
TC_Timer _timer;
96103
};
97104

98105
extern TC_Config* g_pconf;

‎deploy/framework/conf/tarsAdminRegistry/conf/tars.tarsAdminRegistry.config.conf‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,4 @@
7070
patch_wait_timeout = 300
7171
</patch>
7272

73-
<db_reserve>
74-
#stat reserve time(day)
75-
stat_reserve_time = 31
76-
#stat reserve time(day)
77-
property_reserve_time = 31
78-
</db_reserve>
7973
</tars>

‎deploy/framework/conf/tarsproperty/conf/tars.tarsproperty.config.conf‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,9 @@
6565
Interval=10
6666
sql=insert ignore into t_master_property select master_name, property_name, policy from ${TABLE} group by master_name, property_name, policy;
6767
</reapSql>
68+
69+
<db_reserve>
70+
#stat reserve time(day)
71+
property_reserve_time = 31
72+
</db_reserve>
6873
</tars>

‎deploy/framework/conf/tarsstat/conf/tars.tarsstat.config.conf‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,9 @@
6969
charset=utf8
7070
</db1>
7171
</statdb>
72+
73+
<db_reserve>
74+
#stat reserve time(day)
75+
stat_reserve_time = 31
76+
</db_reserve>
7277
</tars>

0 commit comments

Comments
 (0)