optimized number of request to Drupakl API

seminaire/sync_ical_drupal/sync_icals_to_drupal.py

@@ -50,10 +50,14 @@ def is_filtered(event, filter):
 #value written in the revision_log attribute
 def revision_log(url):
     return "ical {}".format(url)
-    
-def process_ical(ical,auth,force_update = False):
+
+#global authentification token, cached to minimize usage of Drupal API
+auth = None
+
+def process_ical(ical,force_update = False):
+    global auth
     log.info("\n************************\nProcessing ical '{}'\n************************".format(ical['name']))
-    teams_ids = utils_teams.get_teams_ids(ical['teams'], auth)
+    teams_ids = utils_teams.get_teams_ids(filter = ical['teams'])
     team_id = teams_ids[ical['default_team']]
     url = ical['ical_url']
     events = utils_ical.get_ical_events(team_id,teams_ids,url, ical['ical_format'])
@@ -114,32 +118,28 @@ def process_ical(ical,auth,force_update = False):
         log.info("Updating {} events".format(len(to_update)))
         log.info("Erasing {} events".format(len(to_delete)))
 
-    
-    utils_drupal.delete_events(auth,to_delete)
+    if not auth:
+        log.info("Opening local session")
+        auth = utils_auth.authenticate()
 
+    utils_drupal.delete_events(auth,to_delete)
     utils_drupal.update_events(auth,to_update)
-
     utils_drupal.create_events(auth,to_create)
-
+        
     log.info("Done ical '{}'".format(ical['name']))
 
 def main():
 
-    log.info("Opening local session")
-    try:
-        auth = utils_auth.authenticate()
-    except Exception as e:
-        log.warn("Failed to authenticate\n {}".format(e))
-        exit(1)
-        
+    global auth
     for ical in config.icals:
         try:
-            process_ical(ical,auth)
+            process_ical(ical)
         except Exception as e:
-            log.warn("Failed to process ical {}\n {}".format(ical['name'], e))
+            log.warning("Failed to process ical {}\n {}".format(ical['name'], e))
             
     log.info("Closing local session")
-    utils_auth.deauthenticate(auth)
+    if auth:
+        utils_auth.deauthenticate(auth)
     
 if __name__ == "__main__":
     main()

seminaire/sync_ical_drupal/utils_drupal.py

@@ -46,22 +46,32 @@ def get_last_url():
         return jsdoc['links']['last']['href']
 
 """parameter is a list of teams ids"""
-def get_events(filter_team_ids, filter_revision_log = None):
+events = None
+
+def retrieve_all_events():
+    global events
+    events = []
     last_url = get_last_url()
-    result = []    
-    while True:        
+    while True:
         jsdoc = get_json(last_url)
-        for event in jsdoc['data']:
-            try:                
-                if event['relationships']['field_equipe']['data']['id'] in filter_team_ids \
-                and (not filter_revision_log or event['attributes']['revision_log'] == filter_revision_log):
-                    result.append({ 'data' : event })
-            except (KeyError ,TypeError):
-                None
+        events = events + jsdoc['data']
         if not 'links' in jsdoc or not 'prev' in jsdoc['links']:
             break
         else:
             last_url = jsdoc['links']['prev']['href']
+
+def get_events(filter_team_ids, filter_revision_log = None):
+    global events
+    if not events:
+        retrieve_all_events()
+    result = []    
+    for event in events:
+        try:                
+            if event['relationships']['field_equipe']['data']['id'] in filter_team_ids \
+               and (not filter_revision_log or event['attributes']['revision_log'] == filter_revision_log):
+                result.append({ 'data' : event })
+        except (KeyError ,TypeError):
+            None
     return result
 
 def delete_events(auth,events):

seminaire/sync_ical_drupal/utils_teams.py

@@ -22,23 +22,27 @@ import utils_auth
 def json_pp(s):
     return json.dumps(s, indent=2, sort_keys=True)
 
-teams_ids = None
+#cached value of the API request
+teams_json = None
 
 """turn a list of nicknames to a mapping from these nickanmes to ids"""
 """if 'filter' is not empty, it is used to filter results"""
 def get_teams_ids(filter, auth = None):    
-    global teams_ids
-    if auth:
-        r = requests.get(
-            config.taxonomy_url,
-            headers = auth['headers'],
-            cookies = auth['cookies']
-        )
-    else:
-        r = requests.get(config.taxonomy_url)
-    if not r:
-        raise Exception("Failed with http status code {} to process request '{}'".format(r.status_code,config.taxonomy_url))
-    teams_json = r.json()
+    #the value is cached to minimize requests to drupal API
+    global teams_json
+    if not teams_json:
+        if auth:
+            r = requests.get(
+                config.taxonomy_url,
+                headers = auth['headers'],
+                cookies = auth['cookies']
+            )
+        else:
+            r = requests.get(config.taxonomy_url)
+            if not r:
+                raise Exception("Failed with http status code {} to process request '{}'".format(r.status_code,config.taxonomy_url))
+        teams_json = r.json()
+
     teams_ids = {}
     #json_pp(teams_json)
     if teams_json and 'data' in teams_json:
@@ -58,7 +62,7 @@ def get_teams_ids(filter, auth = None):
 
 def main():
     m2f_teams = ["M2F","RATIO","DART","MTV","LX"]
-    get_teams_ids(m2f_teams)
+    teams_ids = get_teams_ids(m2f_teams)
     print(teams_ids)
     print("\n")
     print("Id du département:{}".format(teams_ids["M2F"]))