From cacb9d9901d74d18f7ab8f123e17076fddc5edd7 Mon Sep 17 00:00:00 2001 From: Tom Grundy Date: Sat, 19 Nov 2016 14:25:10 -0800 Subject: [PATCH] caver456 suggested overhaul - use python requests instead of external dependency on curl - accomodate special columns Status, Priority, Milestone, HelpWanted - if status starts with DONE or CANCEL, close the issue after creating it - add a comment with full status text as needed - map any additional columns to labels - other edits --- issues.py | 267 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 194 insertions(+), 73 deletions(-) diff --git a/issues.py b/issues.py index 8e2d4b8..b8a0f70 100644 --- a/issues.py +++ b/issues.py @@ -1,84 +1,205 @@ +# import-issues-to-github +# reads a .csv file (first and only argument) and creates github issues accordingly + +# Overall procedure for creating a new issue: +# 1. Will the issue be tied to a milestone? +# YES: does a milestone with the specified name already exist in the repo? +# YES: get the number of that milestone +# NO: create a new milestone and get its number +# NO: go to the next step +# 2. create the issue (and tie it to the specified milestone number if any) +# 3. if status starts with DONE or CANCEL, edit the issue to change its state to 'closed', +# and add a comment with the full status text + +# this works to get all issues for repo 'hello-world' belonging to user 'joe': +# curl -H "Authorization: token aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" https://api.github.com/repos/joe/hello-world/issues + +# this works to create an issue (must use this quote scheme for Windows) +# curl -H "Authorization: token aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -d "{ \"title\":\"New logo\"}" https://api.github.com/repos/joe/hello-world/issues + + +# Mapping from csv to GitHub issue: +# (see GitHub Issues API docs: https://developer.github.com/v3/issues/#create-an-issue ) + +# column name handling +# ----------------------------------------- +# Title 'title' field of POST request +# Body 'body' field of POST request +# Status if text begins with 'DONE' then edit the issue after creating +# to change state to 'closed', and if there is additional text, +# create a comment with full Status text; if 'CANCEL', close +# the issue and add a comment with full Status text; +# otherwise, if not blank, create a comment with the Status text +# HelpWanted if 'yes' then add label 'help wanted', otherwise take no action +# Category add a label with the text of the cell value only +# Milestone if the milestone already exists, use its number; if not, create it first +# ':' added to 'labels' field of POST request + +# example: +# Status,Category,Milestone,Title,Body +# ,bug,docs,Make a README,like it says + +# 11-13-16 TMG change to use python requests instead of external dependency on curl + import os import sys import csv import json -import urllib +import urllib.parse from config import * +import requests import time -column = {} - -def defineTitle(header): - index = 0 - for title in header: - column[title] = index - index+=1 - return - -def printIssue(reader): - issueNumber = 1 - for row in reader: - if issueNumber >= 5: - break - print issueNumber, row[column["Description"]], - issueNumber+=1 - return - -def postRequest(curl): - os.system(curl) - return - -def createIssue(row, projectName): - dictionary = {} - project = projectName - experimentName = row[column["Experiment name"]] - feature = row[column["Feature"]] - testStepNum = row[column["Test Step No"]] - testStepNum = testStepNum.zfill(2) - subject = "QA_%s_%s" %(experimentName, feature) - - explink = urllib.quote(experimentName) - featureLink = urllib.quote(feature) - link = "https://github.com/%s/%s/blob/master/test-cases/integration_test-cases/%s/%s_%s_%s.org" %(organization, project, explink, explink, testStepNum, featureLink) - description = row[column["Description"]] + "\n Test Step Link:\n%s" %(link) - - statusLabel = "Status: " + row[column["Status"]] - severityLabel = "Severity: " + row[column["Severity"]] - categoryLabel = "Category: " + row[column["Category"]] -#assignedByLabel = "Assigned by: " + row[column["Assigned by"]] - releaseNumLabel = "Release Number: " + row[column["Release Number"]] -# dateLabel = "Start Date: " + row[column["Start date"]] - developedByLabel = "Developed By: " + row[column["Developed By"]] - - dictionary["title"] = subject - dictionary["body"] = description - dictionary["labels"] = [statusLabel, severityLabel, categoryLabel, releaseNumLabel, developedByLabel] - jsonString = json.dumps(dictionary) - - curl = """curl -g -i -H 'Authorization: token %s' -d '%s' %s""" %(token, jsonString, githubUrl) - postRequest(curl) - return +# readMilestones: return a dictionary of the repo's existing milestones (keys='title',values='number') +def readMilestones(): + d={} + time.sleep(1) # per GitHub API best practices + response=requests.get(milestonesUrl,headers=headers) + if response.status_code!=200: + print(response.headers) + print(json.loads(response.text)) + msList=json.loads(response.text) + for msDict in msList: +# pd(msDict) + for key in msDict: + d[msDict["title"]]=msDict["number"] + global pauseCount + pauseCount+=1 + return d + + +# createMilestone(title): create a new milestone in the repo with title='title'; +# do not try to return a value; instead, the caller should call readMilestones +# to refresh the index afterwards +def createMilestone(title): + time.sleep(1) # per GitHub API best practices + payload={} + payload["title"]=title + response=requests.post(milestonesUrl,data=json.dumps(payload),headers=headers) + if response.status_code!=201: + print(response.headers) + print(json.loads(response.text)) + print(" Created milestone :",title) + global pauseCount + pauseCount+=1 + + +def createComment(n,body): + time.sleep(1) # per GitHub API best practices + url=issuesUrl+"/"+str(n)+"/comments" + payload={} + payload["body"]=body + response=requests.post(url,data=json.dumps(payload),headers=headers) + if response.status_code!=201: + print(response.headers) + print(json.loads(response.text)) + print(" Created comment for issue ",str(n),":",body) + global pauseCount + pauseCount+=1 + + +def closeIssue(n): + time.sleep(1) # per GitHub API best practices + url=issuesUrl+"/"+str(n) + payload={} + payload["state"]="closed" + response=requests.patch(url,data=json.dumps(payload),headers=headers) + if response.status_code!=200: + print(response.headers) + print(json.loads(response.text)) + print(" Closed issue :",str(n)) + global pauseCount + pauseCount+=1 + + +def createIssue(rowDict): +# pd(rowDict) + d={} + global msIndex + labels=[] + status="" + for key in rowDict: + if key=="Title": + d["title"]=rowDict["Title"] + elif key=="Body": + d["body"]=rowDict["Body"] + elif key=="Status": + status=rowDict["Status"] + elif key=="HelpWanted": + if rowDict["HelpWanted"]=="yes": + labels.append("help wanted") + elif key=="Category": + labels.append(rowDict["Category"]) + elif key=="Milestone": + msTitle=rowDict["Milestone"] + if msTitle!="": + if msTitle not in msIndex: + createMilestone(msTitle) + msIndex=readMilestones() + msNumber=msIndex[msTitle] +# print("milestone:",msTitle,":",msNumber) + d["milestone"]=msNumber + else: + labels.append(key+":"+rowDict[key]) + d["labels"]=labels +# jsonString=json.dumps(dict).replace('"',r'\"') +# print(jsonString) + time.sleep(1) # per GitHub API best practices + response=requests.post(issuesUrl,data=json.dumps(d),headers=headers) + global pauseCount + pauseCount+=1 + if response.status_code!=201: + print(response.headers) + print(json.loads(response.text)) + rd=json.loads(response.text) + issueNumber=rd["number"] + print(" Created issue #"+str(issueNumber)+": "+rd["title"]) + + if status != "": + createComment(issueNumber,status) + if status.startswith("DONE") or status.startswith("CANCEL"): + closeIssue(issueNumber) + +# if there are double-quote issues: +# +# # encoding once doesn't escape double-quotes +# # jsonString = json.dumps(dictionary) +# +# # encoding twice doesn't quite work either: -d "{" becomes -d ""{\" (extra double quote) +# # jsonString = json.dumps(json.dumps(dictionary)) +# +# # instead, encode once then replace all double quotes with backslash-double-quote +# jsonString=json.dumps(dictionary).replace('"',r'\"') + + +def pd(d): + print("\nPrinting dictionary:") + for key in d: + print(key,":",d[key]) + def main(args): - filePointer = open(args[1], 'r') - basename = os.path.basename(args[1]) - projectName = basename.rstrip(".csv") - reader = csv.reader(filePointer, delimiter=',') - header = reader.next() - defineTitle(header) -# printIssue(reader) - rowIndex = 1 - row = reader.next() - while row: - if rowIndex >= 35: - time.sleep(20) - rowIndex = 0 - createIssue(row, projectName) - row = reader.next() - rowIndex+=1 - - filePointer.close() - return + global pauseCount + filePointer = open(args[1], 'r') + rows=csv.DictReader(row for row in filePointer if (not row.startswith('#') and row!="\n")) +# print("row keys and values:") + for row in rows: + if pauseCount>20: + print(" Pausing for 30 seconds to avoid triggering GitHub abuse detection mechanism...") + time.sleep(30) + pauseCount=0 + if row["Title"]!="": +# pd(row) +# print(str(pauseCount)) + createIssue(row) + filePointer.close() + +pauseCount=0 # simple throttling to avoid GitHub abuse detection mechanism +issuesUrl=githubUrl+"/issues" +milestonesUrl=githubUrl+"/milestones" +headers={"Authorization":"token "+token} +msIndex=readMilestones() + if __name__ == "__main__": - main(sys.argv) + main(sys.argv)