Module pagure_events
[hide private]
[frames] | no frames]

Source Code for Module pagure_events

  1  #!/usr/bin/python3 
  2   
  3  import json 
  4  import pprint 
  5  import sys 
  6  import os 
  7  import logging 
  8  import requests 
  9  import re 
 10  import munch 
 11  import subprocess 
 12   
 13  sys.path.append( 
 14      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 15  ) 
 16   
 17  from coprs import db, app, models 
 18  from coprs.logic.coprs_logic import CoprDirsLogic 
 19  from coprs.logic.builds_logic import BuildsLogic 
 20  from coprs.logic.complex_logic import ComplexLogic 
 21  from coprs.logic.packages_logic import PackagesLogic 
 22  from coprs import helpers 
 23   
 24  from urllib.parse import urlparse 
 25   
 26  SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm") 
 27   
 28  log = logging.getLogger(__name__) 
 29  if os.getenv('PAGURE_EVENTS_TESTONLY'): 
 30      ENDPOINT = 'tcp://stg.pagure.io:9940' 
 31  else: 
 32      ENDPOINT = 'tcp://hub.fedoraproject.org:9940' 
 33   
 34  log.setLevel(logging.DEBUG) 
 35  log.info("ENDPOINT = {}".format(ENDPOINT)) 
 36   
 37  TOPICS = {} 
 38  for topic, url in app.config["PAGURE_EVENTS"].items(): 
 39      TOPICS['{0}'.format(topic)] = url 
40 41 -def get_repeatedly(url):
42 log.info("getting url {}".format(url)) 43 for attempt in range(1, 4): 44 r = requests.get(url) 45 if r.status_code == requests.codes.ok: 46 return r.text 47 else: 48 log.error('Bad http status {0} from url {1}, attempt {2}'.format( 49 r.status_code, url, attempt)) 50 # pagure down? 51 return ""
52
53 -class ScmPackage(object):
54 - def __init__(self, db_row):
55 self.source_json_dict = json.loads(db_row.source_json) 56 self.clone_url = self.source_json_dict.get('clone_url') or '' 57 self.committish = self.source_json_dict.get('committish') or '' 58 self.subdirectory = self.source_json_dict.get('subdirectory') or '' 59 60 self.package = ComplexLogic.get_package_by_id_safe(db_row.package_id) 61 self.copr = self.package.copr
62
63 - def build(self, source_dict_update, copr_dir, update_callback, 64 scm_object_type, scm_object_id, scm_object_url, agent_url):
65 66 if self.package.copr_dir.name != copr_dir.name: 67 package = PackagesLogic.get_or_create(copr_dir, self.package.name, self.package) 68 else: 69 package = self.package 70 71 db.session.execute('LOCK TABLE build IN EXCLUSIVE MODE') 72 return BuildsLogic.rebuild_package( 73 package, source_dict_update, copr_dir, update_callback, 74 scm_object_type, scm_object_id, scm_object_url, submitted_by=agent_url)
75 76 @classmethod
77 - def get_candidates_for_rebuild(cls, clone_url):
78 if db.engine.url.drivername == 'sqlite': 79 placeholder = '?' 80 true = '1' 81 else: 82 placeholder = '%s' 83 true = 'true' 84 85 rows = db.engine.execute( 86 """ 87 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id 88 FROM package JOIN copr_dir ON package.copr_dir_id = copr_dir.id 89 WHERE package.source_type = {0} AND 90 package.webhook_rebuild = {1} AND 91 copr_dir.main = {2} AND 92 package.source_json ILIKE {placeholder} 93 """.format(SCM_SOURCE_TYPE, true, true, placeholder=placeholder), '%'+clone_url+'%' 94 ) 95 return [ScmPackage(row) for row in rows]
96 97
98 - def is_dir_in_commit(self, changed_files):
99 if not changed_files: 100 return False 101 102 sm = helpers.SubdirMatch(self.subdirectory) 103 for filename in changed_files: 104 if sm.match(filename): 105 return True 106 107 return False
108
109 110 -def event_info_from_pr_comment(data, base_url):
111 """ 112 Message handler for updated pull-request opened in pagure. 113 Topic: ``*.pagure.pull-request.comment.added`` 114 """ 115 if data['msg']['pullrequest']['status'] != 'Open': 116 log.info('Pull-request not open, discarding.') 117 return False 118 119 if not data['msg']['pullrequest']['comments']: 120 log.info('This is most odd, we\'re not seeing comments.') 121 return False 122 123 last_comment = data['msg']['pullrequest']['comments'][-1] 124 if not last_comment: 125 log.info('Can not access last comment, discarding.') 126 return False 127 128 if not 'comment' in last_comment or '[copr-build]' not in last_comment['comment']: 129 log.info('The [copr-build] is not present in the message.') 130 return False 131 132 return munch.Munch({ 133 'object_id': data['msg']['pullrequest']['id'], 134 'object_type': 'pull-request', 135 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'], 136 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'], 137 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'], 138 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'], 139 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'], 140 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'], 141 'branch_from': data['msg']['pullrequest']['branch_from'], 142 'branch_to': data['msg']['pullrequest']['branch'], 143 'start_commit': data['msg']['pullrequest']['commit_start'], 144 'end_commit': data['msg']['pullrequest']['commit_stop'], 145 'agent': data['msg']['agent'], 146 })
147
148 149 -def event_info_from_pr(data, base_url):
150 """ 151 Message handler for new pull-request opened in pagure. 152 Topic: ``*.pagure.pull-request.new`` 153 """ 154 return munch.Munch({ 155 'object_id': data['msg']['pullrequest']['id'], 156 'object_type': 'pull-request', 157 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'], 158 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'], 159 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'], 160 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'], 161 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'], 162 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'], 163 'branch_from': data['msg']['pullrequest']['branch_from'], 164 'branch_to': data['msg']['pullrequest']['branch'], 165 'start_commit': data['msg']['pullrequest']['commit_start'], 166 'end_commit': data['msg']['pullrequest']['commit_stop'], 167 'agent': data['msg']['agent'], 168 })
169
170 171 -def event_info_from_push(data, base_url):
172 """ 173 Message handler for push event in pagure. 174 Topic: ``*.pagure.git.receive`` 175 """ 176 return munch.Munch({ 177 'object_id': data['msg']['end_commit'], 178 'object_type': 'commit', 179 'base_project_url_path': data['msg']['repo']['url_path'], 180 'base_clone_url_path': data['msg']['repo']['fullname'], 181 'base_clone_url': base_url + data['msg']['repo']['fullname'], 182 'project_url_path': data['msg']['repo']['url_path'], 183 'clone_url_path': data['msg']['repo']['fullname'], 184 'clone_url': base_url + data['msg']['repo']['fullname'], 185 'branch_from': data['msg']['branch'], 186 'branch_to': data['msg']['branch'], 187 'start_commit': data['msg']['start_commit'], 188 'end_commit': data['msg']['end_commit'], 189 'agent': data['msg']['agent'], 190 })
191
192 193 -def git_compare_urls(url1, url2):
194 url1 = re.sub(r'(\.git)?/*$', '', str(url1)) 195 url2 = re.sub(r'(\.git)?/*$', '', str(url2)) 196 o1 = urlparse(url1) 197 o2 = urlparse(url2) 198 return (o1.netloc == o2.netloc and o1.path == o2.path)
199
200 201 -class build_on_fedmsg_loop():
202
203 - def __call__(self, message):
204 pp = pprint.PrettyPrinter(width=120) 205 206 log.debug('Parsing...') 207 data = { 208 'topic': message.topic, 209 'msg': message.body 210 } 211 212 log.info('Got topic: {}'.format(data['topic'])) 213 base_url = TOPICS.get(data['topic']) 214 if not base_url: 215 log.error('Unknown topic {} received. Continuing.') 216 return 217 218 if re.match(r'^.*.pull-request.(new|rebased|updated)$', data['topic']): 219 event_info = event_info_from_pr(data, base_url) 220 elif re.match(r'^.*.pull-request.comment.added$', data['topic']): 221 event_info = event_info_from_pr_comment(data, base_url) 222 else: 223 event_info = event_info_from_push(data, base_url) 224 225 log.info('event_info = {}'.format(pp.pformat(event_info))) 226 227 if not event_info: 228 log.info('Received event was discarded. Continuing.') 229 return 230 231 candidates = ScmPackage.get_candidates_for_rebuild(event_info.base_clone_url) 232 changed_files = set() 233 234 if candidates: 235 raw_commit_url = base_url + event_info.project_url_path + '/raw/' + event_info.start_commit 236 raw_commit_text = get_repeatedly(raw_commit_url) 237 changed_files |= helpers.raw_commit_changes(raw_commit_text) 238 239 if event_info.start_commit != event_info.end_commit: 240 # we want to show changes in start_commit + diff 241 # start_commit..end_commit 242 change_html_url = '{base_url}{project}/c/{start}..{end}'.format( 243 base_url=base_url, 244 project=event_info.project_url_path, 245 start=event_info.start_commit, 246 end=event_info.end_commit) 247 248 change_html_text = get_repeatedly(change_html_url) 249 changed_files |= helpers.pagure_html_diff_changed(change_html_text) 250 251 log.info("changed files: {}".format(", ".join(changed_files))) 252 253 for pkg in candidates: 254 package = '{}/{}(id={})'.format( 255 pkg.package.copr.full_name, 256 pkg.package.name, 257 pkg.package.id 258 ) 259 log.info('Considering pkg package: {}, source_json: {}' 260 .format(package, pkg.source_json_dict)) 261 262 if (git_compare_urls(pkg.clone_url, event_info.base_clone_url) 263 and (not pkg.committish or event_info.branch_to.endswith(pkg.committish)) 264 and pkg.is_dir_in_commit(changed_files)): 265 266 log.info('\t -> accepted.') 267 268 if event_info.object_type == 'pull-request': 269 dirname = pkg.copr.name + ':pr:' + str(event_info.object_id) 270 copr_dir = CoprDirsLogic.get_or_create(pkg.copr, dirname) 271 update_callback = 'pagure_flag_pull_request' 272 scm_object_url = os.path.join(base_url, event_info.project_url_path, 273 'c', str(event_info.end_commit)) 274 else: 275 copr_dir = pkg.copr.main_dir 276 update_callback = 'pagure_flag_commit' 277 scm_object_url = os.path.join(base_url, event_info.base_project_url_path, 278 'c', str(event_info.object_id)) 279 280 if not git_compare_urls(pkg.copr.scm_repo_url, event_info.base_clone_url): 281 update_callback = '' 282 283 source_dict_update = { 284 'clone_url': event_info.clone_url, 285 'committish': event_info.end_commit, 286 } 287 288 try: 289 build = pkg.build( 290 source_dict_update, 291 copr_dir, 292 update_callback, 293 event_info.object_type, 294 event_info.object_id, 295 scm_object_url, 296 "{}user/{}".format(base_url, event_info.agent), 297 ) 298 if build: 299 log.info('\t -> {}'.format(build.to_dict())) 300 except Exception as e: 301 log.error(str(e)) 302 db.session.rollback() 303 else: 304 db.session.commit() 305 else: 306 log.info('\t -> skipping.')
307