Commit @b31a01ea1157f103fbfb92e1ac470d0bc1d91344

윤영준 2024-07-17

first commit of refactoring of data transaction - gps data, attempt to fix problems that 1. data being sent too often, 2. in previous method, the data insert operation happens too often. this commit is trying to fix it by 1. data being sent every 15sec (or more) not every sec, 2. reformatted gps data so that data transaction will cost a lot less bandwidth and 3. insertion operation has been replaced with copy, which is more apt for bulk row appendage operation.

@b31a01ea1157f103fbfb92e1ac470d0bc1d91344

fefe14a

b31a01e

action.py

--- action.py

+++ action.py


 from PIL import Image
 from pothole_model.pothole import pothole
 from datetime import datetime
+import pandas as pd
+from io import StringIO
 import jwt
 
 

     def post(self):
         db = DB()
         data = request.get_json()
-        timestamp = data['timestamp']
-        trip_id = data['trip_id']
-        location_x = float(data['location_x'])
-        location_y = float(data['location_y'])
+        
+        df = pd.DataFrame(data["trip_log"])
+        df["user_id"] = data["user_id"]
+        df["trip_id"] = data["trip_id"]
+        
+        columns = df.columns
+        data_csv_block = df.to_csv(header=False)
+        
+        
         token = request.headers.get('Authorization')
-        #print(token)
-        print(timestamp,location_x,location_y)
-        #print(token)
         if not token:
             return jsonify({'result': 'fail', 'msg': '토큰이 없습니다.'})
         

         db.insert_gps_data(trip_id, location_x, location_y, user_id, timestamp)
         return jsonify({'result': 'success'})
             
-
-         
+            
 @Action.route('/pothole_display')
 class fileUpload(Resource):
     @Action.doc(responses={200: 'Success'})

fefe14a

b31a01e

database/database.py

--- database/database.py

+++ database/database.py


         result=cur.fetchall()
         return result
 
-    def insert_gps_data(self, trip_id, location_x, location_y,user_id, timestamp):
+    def insert_gps_data(self, csv_block, columns):
         cur = self.conn.cursor()
         
-        print(trip_id, location_x, location_y, user_id)
-        trip_id = trip_id['current'] 
-        cur.execute(f'''
-        INSERT INTO "{self.schema}".gps_data (timestamp, trip_id, location_x, location_y,user_id)
-        VALUES (%s, %s, %s, %s,%s);
-        ''', (timestamp, trip_id, location_x, location_y,user_id))
+        data = StringIO(csv_block)
         
+        cur.copy_from(data, 'gps_data', sep=',' columns = columns)
+        self.conn.commit()
         cur.close()
-        return True        
+        return True             
 
 
     

Add a comment

Open 0
Closed 0

List

...	...	@@ -12,6 +12,8 @@
12	12	from PIL import Image
13	13	from pothole_model.pothole import pothole
14	14	from datetime import datetime
	15	+import pandas as pd
	16	+from io import StringIO
15	17	import jwt
16	18
17	19
...	...	@@ -164,14 +166,16 @@
164	166	def post(self):
165	167	db = DB()
166	168	data = request.get_json()
167		- timestamp = data['timestamp']
168		- trip_id = data['trip_id']
169		- location_x = float(data['location_x'])
170		- location_y = float(data['location_y'])
	169	+
	170	+ df = pd.DataFrame(data["trip_log"])
	171	+ df["user_id"] = data["user_id"]
	172	+ df["trip_id"] = data["trip_id"]
	173	+
	174	+ columns = df.columns
	175	+ data_csv_block = df.to_csv(header=False)
	176	+
	177	+
171	178	token = request.headers.get('Authorization')
172		- #print(token)
173		- print(timestamp,location_x,location_y)
174		- #print(token)
175	179	if not token:
176	180	return jsonify({'result': 'fail', 'msg': '토큰이 없습니다.'})
177	181
...	...	@@ -185,8 +189,7 @@
185	189	db.insert_gps_data(trip_id, location_x, location_y, user_id, timestamp)
186	190	return jsonify({'result': 'success'})
187	191
188		-
189		-
	192	+
190	193	@Action.route('/pothole_display')
191	194	class fileUpload(Resource):
192	195	@Action.doc(responses={200: 'Success'})

...	...	@@ -198,18 +198,15 @@
198	198	result=cur.fetchall()
199	199	return result
200	200
201		- def insert_gps_data(self, trip_id, location_x, location_y,user_id, timestamp):
	201	+ def insert_gps_data(self, csv_block, columns):
202	202	cur = self.conn.cursor()
203	203
204		- print(trip_id, location_x, location_y, user_id)
205		- trip_id = trip_id['current']
206		- cur.execute(f'''
207		- INSERT INTO "{self.schema}".gps_data (timestamp, trip_id, location_x, location_y,user_id)
208		- VALUES (%s, %s, %s, %s,%s);
209		- ''', (timestamp, trip_id, location_x, location_y,user_id))
	204	+ data = StringIO(csv_block)
210	205
	206	+ cur.copy_from(data, 'gps_data', sep=',' columns = columns)
	207	+ self.conn.commit()
211	208	cur.close()
212		- return True
	209	+ return True
213	210
214	211
215	212

Delete comment