Skip to content

adds threadpooling for individual team syncing #101

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 15, 2021
Merged

adds threadpooling for individual team syncing #101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a13e8e4
adds threadpooling for individual team syncing
Oct 14, 2021
c94936e
adding some debugging but ultimately everything was fine
Oct 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 43 additions & 29 deletions app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import threading
import sys
import traceback
from concurrent.futures import ThreadPoolExecutor

from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
Expand Down Expand Up @@ -69,7 +70,8 @@ def sync_team(client=None, owner=None, team_id=None, slug=None):
directory_members = directory_group_members(group=directory_group)
except Exception as e:
directory_members = []
print(e)
traceback.print_exc(file=sys.stderr)

team_members = github_team_members(
client=client, owner=owner, team_id=team_id, attribute=USER_SYNC_ATTRIBUTE
)
Expand Down Expand Up @@ -107,7 +109,7 @@ def directory_group_members(group=None):
group_members = [member for member in members]
except Exception as e:
group_members = []
print(e)
traceback.print_exc(file=sys.stderr)
return group_members


Expand Down Expand Up @@ -274,37 +276,49 @@ def sync_all_teams():
Lookup teams in a GitHub org and synchronize all teams with your user directory
:return:
"""

print(f'Syncing all teams: {time.strftime("%A, %d. %B %Y %I:%M:%S %p")}')

installations = get_app_installations()
custom_map = load_custom_map()
for i in installations():
print("========================================================")
print(f"## Processing Organization: {i.account['login']}")
print("========================================================")
with app.app_context() as ctx:
try:
gh = GitHubApp(ctx.push())
client = gh.app_installation(installation_id=i.id)
org = client.organization(i.account["login"])
for team in org.teams():
try:
if SYNCMAP_ONLY and team.slug not in custom_map:
print(f"skipping team {team.slug} - not in sync map")
continue
sync_team(
client=client,
owner=org.login,
team_id=team.id,
slug=team.slug,
futures = []
with ThreadPoolExecutor(max_workers=10) as exe:
for i in installations():
print("========================================================")
print(f"## Processing Organization: {i.account['login']}")
print("========================================================")
with app.app_context() as ctx:
try:
gh = GitHubApp(ctx.push())
client = gh.app_installation(installation_id=i.id)
org = client.organization(i.account["login"])
for team in org.teams():
futures.append(
exe.submit(sync_team_helper, team, custom_map, client, org)
)
except Exception as e:
print(f"Organization: {org.login}")
print(f"Unable to sync team: {team.slug}")
print(f"DEBUG: {e}")
except Exception as e:
print(f"DEBUG: {e}")
finally:
ctx.pop()
except Exception as e:
print(f"DEBUG: {e}")
finally:
ctx.pop()
for future in futures:
future.result()


def sync_team_helper(team, custom_map, client, org):
try:
if SYNCMAP_ONLY and team.slug not in custom_map:
print(f"skipping team {team.slug} - not in sync map")
return
sync_team(
client=client,
owner=org.login,
team_id=team.id,
slug=team.slug,
)
except Exception as e:
print(f"Organization: {org.login}")
print(f"Unable to sync team: {team.slug}")
print(f"DEBUG: {e}")


thread = threading.Thread(target=sync_all_teams)
Expand Down
38 changes: 23 additions & 15 deletions githubapp/ldap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import os
import traceback
import sys
import json
import logging
from ldap3 import Server, Connection, ALL
Expand Down Expand Up @@ -73,18 +75,23 @@ def get_group_members(self, group_name):
try:
member_dn = self.get_user_info(user=member)
# pprint(member_dn)
username = str(
member_dn["attributes"][self.LDAP_USER_ATTRIBUTE][0]
).casefold()
email = str(
member_dn["attributes"][self.LDAP_USER_MAIL_ATTRIBUTE][
0
]
).casefold()
user_info = {"username": username, "email": email}
member_list.append(user_info)
if (
member_dn
and member_dn["attributes"]
and member_dn["attributes"][self.LDAP_USER_ATTRIBUTE]
):
username = str(
member_dn["attributes"][self.LDAP_USER_ATTRIBUTE][0]
).casefold()
email = str(
member_dn["attributes"][
self.LDAP_USER_MAIL_ATTRIBUTE
][0]
).casefold()
user_info = {"username": username, "email": email}
member_list.append(user_info)
except Exception as e:
print(e)
traceback.print_exc(file=sys.stderr)
return member_list

def get_user_info(self, user=None):
Expand All @@ -106,9 +113,10 @@ def get_user_info(self, user=None):
search_filter=self.LDAP_USER_FILTER.replace("{username}", user),
attributes=["*"],
)
data = json.loads(self.conn.entries[0].entry_to_json())
return data
if len(self.conn.entries) > 0:
data = json.loads(self.conn.entries[0].entry_to_json())
return data
except Exception as e:
print(e)
traceback.print_exc(file=sys.stderr)
except Exception as e:
print(e)
traceback.print_exc(file=sys.stderr)