From b234a5dc558f58de30fd05923fb022dc95d1f636 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 4 Apr 2012 13:13:38 -0400 Subject: [PATCH 001/141] Fix typo causing attachment flag changes to not get filtered properly --- dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard.php b/dashboard.php index 4e60abb..e99fb74 100644 --- a/dashboard.php +++ b/dashboard.php @@ -148,7 +148,7 @@ function column( &$reasons ) { $reasons[ $row['bug'] ][] = 'review'; $filterComments[ $row['attachment'] ][] = $row['comment']; - $filterFlags[ $row['attachment'] ][] = array( "{$row['flag']}?({$row['authoremail']})", "{$type}" . ($row['granted'] ? '+' : '-') ); + $filterFlags[ $row['attachment'] ][] = array( "{$row['flag']}?({$row['authoremail']})", "{$row['flag']}" . ($row['granted'] ? '+' : '-') ); } $result = loadTable( 'requests' ); From b51bc867fd09f148c8108d6f7a0d6722aac59bdb Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 6 Apr 2012 20:36:25 -0400 Subject: [PATCH 002/141] Add bugtags support for dependency tree views --- bugtags.user.js | 104 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 20 deletions(-) diff --git a/bugtags.user.js b/bugtags.user.js index 24ffb4d..2fd7663 100644 --- a/bugtags.user.js +++ b/bugtags.user.js @@ -22,7 +22,7 @@ function getUser() { return null; } -function getBugNumbers() { +function getListBugNumbers() { var bugnumbers = new Array(); var table = document.getElementsByClassName( "bz_buglist" ); @@ -38,7 +38,47 @@ function getBugNumbers() { return bugnumbers; } -function insertBugTags( user, bugnumbers ) { +function getTreeBugNumbers() { + var bugnumbers = new Array(); + + var tree = document.getElementsByClassName( "tree" ); + if (tree.length != 1) { + return bugnumbers; + } + tree = tree[0]; + + var nodes = tree.getElementsByClassName( "summ_deep" ); + for (var i = 0; i < nodes.length; i++) { + bugnumbers.push( nodes[i].id ); + } + nodes = tree.getElementsByClassName( "summ" ); + for (var i = 0; i < nodes.length; i++) { + bugnumbers.push( nodes[i].id ); + } + return bugnumbers; +} + +function buildTag( response, bugnumber ) { + var color = 'blue'; + var tags = '+'; + if (response[ bugnumber ]) { + tags = response[ bugnumber ].join( ", " ); + if (tags.charAt( 0 ) == '!') { + tags = tags.substring( 1 ); + color = 'red'; + } + } + var tag = document.createElement( 'a' ); + tag.id = 'bugmash' + bugnumber; + tag.href = '#'; + tag.style.fontSize = 'smaller'; + tag.style.color = color; + tag.textContent = tags; + tag.addEventListener( 'click', updateBugTag, false ); + return tag; +} + +function insertListBugTags( user, bugnumbers ) { var reqData = new FormData(); reqData.append( "user", user ); reqData.append( "action", "get" ); @@ -54,23 +94,8 @@ function insertBugTags( user, bugnumbers ) { for (var i = 0; i < rows.length; i++) { var row = rows[i]; var bugnumber = row.id.substring( 1 ); + var tag = buildTag( response, bugnumber ); var cell = row.cells[ row.cells.length - 1 ]; - var color = 'blue'; - var tags = '+'; - if (response[ bugnumber ]) { - tags = response[ bugnumber ].join( ", " ); - if (tags.charAt( 0 ) == '!') { - tags = tags.substring( 1 ); - color = 'red'; - } - } - var tag = document.createElement( 'a' ); - tag.id = 'bugmash' + bugnumber; - tag.href = '#'; - tag.style.fontSize = 'smaller'; - tag.style.color = color; - tag.textContent = tags; - tag.addEventListener( 'click', updateBugTag, false ); cell.insertBefore( tag, cell.firstChild ); } }, @@ -82,6 +107,41 @@ function insertBugTags( user, bugnumbers ) { }); } +function insertTreeBugTags( user, bugnumbers ) { + var reqData = new FormData(); + reqData.append( "user", user ); + reqData.append( "action", "get" ); + reqData.append( "bugs", bugnumbers.join( "," ) ); + + GM_xmlhttpRequest({ + method: "POST", + url: TAGS_SERVER, + data: reqData, + onload: function( res ) { + var response = res.responseJSON; + var nodes = document.getElementsByClassName( "tree" )[0].getElementsByClassName( "summ_deep" ); + for (var i = 0; i < nodes.length; i++) { + var node = nodes[i]; + var bugnumber = node.id; + var tag = buildTag( response, bugnumber ); + node.insertBefore( tag, node.lastElementChild ); + } + nodes = document.getElementsByClassName( "tree" )[0].getElementsByClassName( "summ" ); + for (var i = 0; i < nodes.length; i++) { + var node = nodes[i]; + var bugnumber = node.id; + var tag = buildTag( response, bugnumber ); + node.insertBefore( tag, node.lastElementChild ); + } + }, + onerror: function( res ) { + GM_log( "Error fetching bug tags!" ); + GM_log( res.statusText ); + GM_log( res.responseText ); + } + }); +} + function updateBugTag( e ) { e.preventDefault(); @@ -133,8 +193,12 @@ function updateBugTag( e ) { var user = getUser(); if (user) { - var bugnumbers = getBugNumbers(); + var bugnumbers = getListBugNumbers(); + if (bugnumbers.length > 0) { + insertListBugTags( user, bugnumbers ); + } + bugnumbers = getTreeBugNumbers(); if (bugnumbers.length > 0) { - insertBugTags( user, bugnumbers ); + insertTreeBugTags( user, bugnumbers ); } } From 2daff3cdc6e8e3999c6d5fac389f88e7275967f6 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sat, 7 Apr 2012 23:49:13 -0500 Subject: [PATCH 003/141] Add bug titles as link titles during linkification --- dashboard.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dashboard.php b/dashboard.php index e99fb74..6fc69af 100644 --- a/dashboard.php +++ b/dashboard.php @@ -96,11 +96,16 @@ function stripWhitespace( $stuff ) { return preg_replace( '/\s/', '', $stuff ); } +function buglink( $prefix, $bug ) { + global $_BASE_URL, $meta_titles; + return '' . $prefix . $bug . ''; +} + function linkify( $text, $bug ) { global $_BASE_URL; $text = preg_replace( '#(https?://\S+)#i', '$1', $text ); - $text = preg_replace( '/(bug\s+)(\d+)/i', '$1$2', $text ); - $text = preg_replace( '/(bug-)(\d+)/i', '$1$2', $text ); + $text = preg_replace( '/(bug\s+)(\d+)/ie', 'buglink(\'\\1\', \'\\2\')', $text ); + $text = preg_replace( '/(bug-)(\d+)/ie', 'buglink(\'\\1\', \'\\2\')', $text ); $text = preg_replace( '/(Attachment #?)(\d+)/i', '$1$2', $text ); return $text; } From e9e7a706a8d093b37cae81572383ab2b7bf2ce89 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 7 May 2012 10:00:36 -0400 Subject: [PATCH 004/141] Add linkification for bugnumbers in the depends/blocks fields --- dashboard.php | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dashboard.php b/dashboard.php index 6fc69af..33bfde7 100644 --- a/dashboard.php +++ b/dashboard.php @@ -110,6 +110,14 @@ function linkify( $text, $bug ) { return $text; } +function buglinkify( $field, $text ) { + global $_BASE_URL; + if ($field === 'Depends on' || $field === 'Blocks') { + $text = preg_replace( '/(\d+)/e', 'buglink(\'\', \'\\1\')', $text ); + } + return $text; +} + function column( &$reasons ) { if (array_search( 'review', $reasons ) !== FALSE) { return 0; @@ -211,8 +219,8 @@ function column( &$reasons ) { ($hide ? ' style="display: none"' : ''), $row['id'], linkify( escapeHTML( $row['field'] ), $row['bug'] ), - escapeHTML( $row['oldval'] ), - escapeHTML( $row['newval'] ) ) . "\n"; + buglinkify( $row['field'], escapeHTML( $row['oldval'] ) ), + buglinkify( $row['field'], escapeHTML( $row['newval'] ) ) ) . "\n"; $reasons[ $row['bug'] ][] = $row['reason']; } From 57fc7351c9e698b91d53d4a9dd21af3e03e224cb Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 24 May 2012 13:52:50 -0400 Subject: [PATCH 005/141] Guard against passing an empty needle to strpos --- dashboard.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dashboard.php b/dashboard.php index 33bfde7..4f7a252 100644 --- a/dashboard.php +++ b/dashboard.php @@ -234,7 +234,8 @@ function column( &$reasons ) { if (isset( $filterComments[ $matches[1] ] )) { foreach ($filterComments[ $matches[1] ] AS $filterComment) { // strip whitespace before comparison because sometimes the emails are formatted differently. stupid bugzilla - if (strpos( stripWhitespace( $row['comment'] ), stripWhitespace( $filterComment ) ) !== FALSE) { + $strippedComment = stripWhitespace( $filterComment ); + if (strlen( $strippedComment ) > 0 && strpos( stripWhitespace( $row['comment'] ), $strippedComment ) !== FALSE) { $hide = true; break; } From 821d2c5ff5e9c84a2df51fe0dadd1a0cecf4a6a6 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 4 Jun 2012 23:08:24 -0400 Subject: [PATCH 006/141] Update saveDependencyChanges to deal with multiple change tables in a single email --- filter.php | 56 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/filter.php b/filter.php index 8ef4819..ca8b7f5 100644 --- a/filter.php +++ b/filter.php @@ -288,35 +288,37 @@ function saveComments( $bug, $date, $reason, &$mailString ) { } function saveDependencyChanges( $bug, $date, $reason, &$mailString ) { - $matches = array(); - if (preg_match( '/Bug (\d+) depends on bug (\d+), which changed state./', $mailString, $matches ) == 0) { - return false; - } - if (strcmp( $bug, $matches[1] ) != 0) { - fail( 'Dependency email did not match bug number' ); - } - $dependentBug = $matches[2]; + $offset = 0; + while (true) { + $matches = array(); + if (preg_match( '/Bug (\d+) depends on bug (\d+), which changed state./', $mailString, $matches, 0, $offset ) == 0) { + break; + } + if (strcmp( $bug, $matches[1] ) != 0) { + fail( 'Dependency email did not match bug number' ); + } + $dependentBug = $matches[2]; - // in this case we don't know the list of field names ahead of time, just that it will be one or more Status - // and Resolution fields. Since these won't line wrap, we can just do a simpler version of parseChangeTable - $matches = array(); - $matchCount = preg_match_all( "/\n *What *\|Old Value *\|New Value\n-*\n(.*?)\n\n/s", $mailString, $matches, PREG_PATTERN_ORDER ); - if ($matchCount != 1) { - fail( 'Found ' . $matchCount . ' change tables in a dependency change email' ); - } - $tableRows = explode( "\n", $matches[1][0] ); - $fields = array(); - $oldvals = array(); - $newvals = array(); - foreach ($tableRows AS $row) { - list( $field, $oldval, $newval ) = explode( '|', $row ); - $fields[] = 'depbug-' . $dependentBug . '-' . trim( $field ); - $oldvals[] = trim( $oldval ); - $newvals[] = trim( $newval ); + // in this case we don't know the list of field names ahead of time, just that it will be one or more Status + // and Resolution fields. Since these won't line wrap, we can just do a simpler version of parseChangeTable + $matches = array(); + if (preg_match( "/\n *What *\|Old Value *\|New Value\n-*\n(.*?)\n\n/s", $mailString, $matches, PREG_OFFSET_CAPTURE, $offset ) == 0) { + fail( 'Did not find change table corresponding to dependency change for bug ' . $dependentBug ); + } + $tableRows = explode( "\n", $matches[1][0] ); + $fields = array(); + $oldvals = array(); + $newvals = array(); + foreach ($tableRows AS $row) { + list( $field, $oldval, $newval ) = explode( '|', $row ); + $fields[] = 'depbug-' . $dependentBug . '-' . trim( $field ); + $oldvals[] = trim( $oldval ); + $newvals[] = trim( $newval ); + } + insertChanges( $bug, $date, $reason, $fields, $oldvals, $newvals ); + $offset = $matches[1][1] + strlen( $matches[1][0] ); } - insertChanges( $bug, $date, $reason, $fields, $oldvals, $newvals ); - - return true; + return ($offset > 0); } function prepare( $query ) { From 6a6574b1b57f34d43b5046c5516b513f4bd6e4c6 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 7 Jun 2012 16:23:41 -0400 Subject: [PATCH 007/141] Deal with roc's multi-line author info --- filter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.php b/filter.php index ca8b7f5..57e651d 100644 --- a/filter.php +++ b/filter.php @@ -397,7 +397,7 @@ function updateMetadata( $date ) { // this may cancel something we don't have a record of; if so, ignore success(); } else { - if (preg_match( "/\n(.*) <(.*)> has (?:not )?granted/", $mailString, $matches ) == 0) { + if (preg_match( "/\n(.*) <(.*)> has (?:not )?granted/sU", $mailString, $matches ) == 0) { fail( 'Unable to determine author of review' ); } $author = $matches[1]; From d2ab0db715d51d41b2610c3af2bb923b90d85e6c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sat, 9 Jun 2012 14:14:30 -0400 Subject: [PATCH 008/141] Revert "Deal with roc's multi-line author info" This reverts commit 6a6574b1b57f34d43b5046c5516b513f4bd6e4c6, apparently it doesn't work as well as I thought. --- filter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.php b/filter.php index 57e651d..ca8b7f5 100644 --- a/filter.php +++ b/filter.php @@ -397,7 +397,7 @@ function updateMetadata( $date ) { // this may cancel something we don't have a record of; if so, ignore success(); } else { - if (preg_match( "/\n(.*) <(.*)> has (?:not )?granted/sU", $mailString, $matches ) == 0) { + if (preg_match( "/\n(.*) <(.*)> has (?:not )?granted/", $mailString, $matches ) == 0) { fail( 'Unable to determine author of review' ); } $author = $matches[1]; From 2827f4b9dd03a7792dc65cdac8b013ecb5cffd68 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 13 Jun 2012 09:49:01 -0400 Subject: [PATCH 009/141] Add some more known issueS --- README.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.html b/README.html index e93be66..41beec1 100644 --- a/README.html +++ b/README.html @@ -81,6 +81,8 @@

Known issues

  • The code to strikethrough obsoleted review requests doesn't seem to be taking effect. Not sure where in the code this bug is, haven't really looked at it.
  • When concurrently editing tags via both the user script and the dashboard, it is easy to accidentally clobber tag changes. This should be detected and handled better.
    • +
  • Bugzilla comments which contain "--" on a single line will be prematurely terminated, since that is taken as the end-of-comment marker.
    • +
  • Review authors whose name/email wraps onto a second line in the email are not handled properly.

Areas for improvement

From ed22b2e8096fe0a044c63ec59b61fb13966b6c71 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 22 Jun 2012 15:25:22 -0400 Subject: [PATCH 010/141] Deal with some weird emails which have a changed-fields header but just a comment --- filter.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/filter.php b/filter.php index ca8b7f5..f03f83e 100644 --- a/filter.php +++ b/filter.php @@ -256,6 +256,11 @@ function saveChanges( $bug, $date, $reason, &$mailString ) { $matches = array(); $matchCount = preg_match_all( "/\n( *What *\|Removed *\|Added\n-*\n.*?)\n\n/s", $mailString, $matches, PREG_PATTERN_ORDER | PREG_OFFSET_CAPTURE ); if ($matchCount == 0) { + // we might end up here in some cases if the only "field" that changed + // is a comment privacy flag + if (count( $fields ) == 1 && strpos( $fields[0], " is private" ) !== FALSE) { + return $ret; + } fail( 'No change table' ); } $tableRows = $matches[1][0][0]; From 1f4f59dddf1bed45542bcb15b3ce3022a9013159 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 23 Aug 2012 10:33:36 -0400 Subject: [PATCH 011/141] Update scraper to deal with changes in X-Bugzilla-Changed-Fields header that will be reverted in bug 785063. This also allows having no change table on new bug notifications. --- filter.php | 104 ++++++++++++++++++++++++++++------------------------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/filter.php b/filter.php index f03f83e..e750a61 100644 --- a/filter.php +++ b/filter.php @@ -138,59 +138,62 @@ function normalizeReason( $reason, $watchReason ) { } } +$_FIELD_NAME_MAP = array( + 'assigned_to' => 'Assignee', + 'attachments.description' => 'Attachment #...... description', + 'attachments.isobsolete' => 'Attachment #...... is obsolete', + 'attachments.ispatch' => 'Attachment #...... is patch', + 'blocked' => 'Blocks', + 'bug_id' => 'Bug ID', + 'bug_severity' => 'Severity', + 'bug_status' => 'Status', + 'cc' => 'CC', + 'cf_crash_signature' => 'Crash Signature', + 'cf_last_resolved' => 'Last Resolved', + 'cf_blocking_fennec' => 'tracking-fennec', + 'classification' => 'Classification', + 'component' => 'Component', + 'dependson' => 'Depends on', + 'everconfirmed' => 'Ever confirmed', + 'flagtypes.name' => 'Attachment #...... Flags', + 'keywords' => 'Keywords', + 'op_sys' => 'OS', + 'priority' => 'Priority', + 'product' => 'Product', + 'reporter' => 'Reporter', + 'rep_platform' => 'Hardware', + 'resolution' => 'Resolution', + 'short_desc' => 'Summary', + 'status_whiteboard' => 'Whiteboard', + 'target_milestone' => 'Target Milestone', + 'version' => 'Version' +); + function normalizeFieldList( $fieldString ) { + global $_FIELD_NAME_MAP; + $words = array_filter( explode( ' ', $fieldString ) ); $fields = array(); - $currentField = ''; for ($i = 0; $i < count( $words ); $i++) { $word = $words[ $i ]; - if ($word == 'Attachment' /* Created|#abcdef */) { - if ($i + 1 >= count( $words )) { - fail( 'Unrecognized field list (1): ' . print_r( $words, true ) ); - } - $word .= ' ' . $words[ ++$i ]; - if ($words[ $i ] == 'Created') { - // ignore "Attachment Created" in the field list since it doesn't have - // a corresponding entry in the field table - continue; - } else { - /* Flags|is|mime */ - if ($i + 1 >= count( $words )) { - fail( 'Unrecognized field list (2): ' . print_r( $words, true ) ); - } - $word .= ' ' . $words[ ++$i ]; - if ($words[ $i ] == 'is' /* obsolete */ || $words[ $i ] == 'mime' /* type */) { - if ($i + 1 >= count( $words )) { - fail( 'Unrecognized field list (3): ' . print_r( $words, true ) ); - } - $word .= ' ' . $words[ ++$i ]; - } - } - } else if ($word == 'Depends' /* On */ - || $word == 'Target' /* Milestone */ - || $word == 'Ever' /* Confirmed */ - || $word == 'Crash' /* Signature */ - || $word == 'See' /* Also */ - || $word == 'Last' /* Resolved */) - { - if ($i + 1 >= count( $words )) { - fail( 'Unrecognized field list (4): ' . print_r( $words, true ) ); - } - $word .= ' ' . $words[ ++$i ]; - } else if ($word == 'Status') { - if ($i + 1 < count( $words ) && $words[ $i + 1 ] == 'Whiteboard') { - $word .= ' '. $words[ ++$i ]; - } - } else if ($word == 'Comment') { - if ($i + 3 < count( $words) && $words[ $i + 2 ] == 'is' && $words[ $i + 3 ] == 'private') { - $word .= ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ]; - } + if (strpos( $word, 'cf_status_' ) === 0) { + $fields[] = 'status-' . substr( $word, strlen( 'cf_status_' ) ); + } else if (strpos( $word, 'cf_tracking_' ) === 0) { + $fields[] = 'tracking-' . substr( $word, strlen( 'cf_tracking_' ) ); + } else if (strcmp( $word, 'attachments.created' ) == 0) { + continue; // no entry in the table + } else { + $field = $_FIELD_NAME_MAP[ $word ]; + $fields[] = $field; } - $fields[] = $word; } return $fields; } +function stripDigits( $value ) { + return preg_replace( '/(\d){6}/', '......', $value ); +} + function parseChangeTable( $fields, $rows ) { // get widths to avoid dying on new/old values with pipe characters $columns = explode( '|', $rows[0] ); @@ -208,7 +211,7 @@ function parseChangeTable( $fields, $rows ) { continue; } $matchedStart = false; - if (strpos( $fields[ $ixField ], $col1 ) === 0) { + if (strpos( $fields[ $ixField ], stripDigits( $col1 ) ) === 0) { $matchedStart = true; if ($ixField > 0) { $oldvals[] = trim( $oldval ); @@ -217,7 +220,7 @@ function parseChangeTable( $fields, $rows ) { $oldval = $col2; $newval = $col3; } - if (strpos( $fields[ $ixField ], $col1 ) === strlen( $fields[ $ixField ] ) - strlen( $col1 )) { + if (strpos( $fields[ $ixField ], stripDigits( $col1 ) ) === strlen( $fields[ $ixField ] ) - strlen( $col1 )) { if (! $matchedStart) { $oldval .= $col2; $newval .= $col3; @@ -246,7 +249,7 @@ function insertChanges( $bug, $date, $reason, &$fields, &$oldvals, &$newvals ) { } } -function saveChanges( $bug, $date, $reason, &$mailString ) { +function saveChanges( $bug, $date, $reason, &$mailString, $allowNoChangeTable ) { $ret = 0; $fields = normalizeFieldList( getField( 'changed-fields' ) ); if (count( $fields ) == 0) { @@ -261,6 +264,9 @@ function saveChanges( $bug, $date, $reason, &$mailString ) { if (count( $fields ) == 1 && strpos( $fields[0], " is private" ) !== FALSE) { return $ret; } + if ($allowNoChangeTable) { + return $ret; + } fail( 'No change table' ); } $tableRows = $matches[1][0][0]; @@ -441,12 +447,12 @@ function updateMetadata( $date ) { $title = trim( $matches[1] ); $author = getField( 'who' ); $matches = array(); - if (preg_match( "/Bug #: .*?\n\n\n(.*\n\n)?-- \n/s", $mailString, $matches ) == 0) { + if (preg_match( "/Bug ID: .*?\n\n(.*\n\n)?-- \n/s", $mailString, $matches ) == 0) { fail( 'No description' ); } $desc = trim( $matches[1] ); - $extracted = saveChanges( $bug, $date, $reason, $desc ); + $extracted = saveChanges( $bug, $date, $reason, $desc, true ); $desc = trim( substr( $desc, $extracted ) ); $stmt = prepare( 'INSERT INTO newbugs (bug, stamp, reason, title, author, description) VALUES (?, ?, ?, ?, ?, ?)' ); @@ -459,7 +465,7 @@ function updateMetadata( $date ) { } else if ($type == 'changed') { $reason = normalizeReason( getField( 'reason' ), getField( 'watch-reason' ) ); - if (saveChanges( $bug, $date, $reason, $mailString ) == 0) { + if (saveChanges( $bug, $date, $reason, $mailString, false ) == 0) { saveDependencyChanges( $bug, $date, $reason, $mailString ); } saveComments( $bug, $date, $reason, $mailString ); From a7e66c6a1d8fde85477834571d70cebf674d9793 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 24 Aug 2012 08:27:52 -0400 Subject: [PATCH 012/141] Revert "Update scraper to deal with changes in X-Bugzilla-Changed-Fields header that will be reverted in bug 785063. This also allows having no change table on new bug notifications." This reverts commit 1f4f59dddf1bed45542bcb15b3ce3022a9013159, since the bug was fixed upstream. --- filter.php | 104 +++++++++++++++++++++++++---------------------------- 1 file changed, 49 insertions(+), 55 deletions(-) diff --git a/filter.php b/filter.php index e750a61..f03f83e 100644 --- a/filter.php +++ b/filter.php @@ -138,62 +138,59 @@ function normalizeReason( $reason, $watchReason ) { } } -$_FIELD_NAME_MAP = array( - 'assigned_to' => 'Assignee', - 'attachments.description' => 'Attachment #...... description', - 'attachments.isobsolete' => 'Attachment #...... is obsolete', - 'attachments.ispatch' => 'Attachment #...... is patch', - 'blocked' => 'Blocks', - 'bug_id' => 'Bug ID', - 'bug_severity' => 'Severity', - 'bug_status' => 'Status', - 'cc' => 'CC', - 'cf_crash_signature' => 'Crash Signature', - 'cf_last_resolved' => 'Last Resolved', - 'cf_blocking_fennec' => 'tracking-fennec', - 'classification' => 'Classification', - 'component' => 'Component', - 'dependson' => 'Depends on', - 'everconfirmed' => 'Ever confirmed', - 'flagtypes.name' => 'Attachment #...... Flags', - 'keywords' => 'Keywords', - 'op_sys' => 'OS', - 'priority' => 'Priority', - 'product' => 'Product', - 'reporter' => 'Reporter', - 'rep_platform' => 'Hardware', - 'resolution' => 'Resolution', - 'short_desc' => 'Summary', - 'status_whiteboard' => 'Whiteboard', - 'target_milestone' => 'Target Milestone', - 'version' => 'Version' -); - function normalizeFieldList( $fieldString ) { - global $_FIELD_NAME_MAP; - $words = array_filter( explode( ' ', $fieldString ) ); $fields = array(); + $currentField = ''; for ($i = 0; $i < count( $words ); $i++) { $word = $words[ $i ]; - if (strpos( $word, 'cf_status_' ) === 0) { - $fields[] = 'status-' . substr( $word, strlen( 'cf_status_' ) ); - } else if (strpos( $word, 'cf_tracking_' ) === 0) { - $fields[] = 'tracking-' . substr( $word, strlen( 'cf_tracking_' ) ); - } else if (strcmp( $word, 'attachments.created' ) == 0) { - continue; // no entry in the table - } else { - $field = $_FIELD_NAME_MAP[ $word ]; - $fields[] = $field; + if ($word == 'Attachment' /* Created|#abcdef */) { + if ($i + 1 >= count( $words )) { + fail( 'Unrecognized field list (1): ' . print_r( $words, true ) ); + } + $word .= ' ' . $words[ ++$i ]; + if ($words[ $i ] == 'Created') { + // ignore "Attachment Created" in the field list since it doesn't have + // a corresponding entry in the field table + continue; + } else { + /* Flags|is|mime */ + if ($i + 1 >= count( $words )) { + fail( 'Unrecognized field list (2): ' . print_r( $words, true ) ); + } + $word .= ' ' . $words[ ++$i ]; + if ($words[ $i ] == 'is' /* obsolete */ || $words[ $i ] == 'mime' /* type */) { + if ($i + 1 >= count( $words )) { + fail( 'Unrecognized field list (3): ' . print_r( $words, true ) ); + } + $word .= ' ' . $words[ ++$i ]; + } + } + } else if ($word == 'Depends' /* On */ + || $word == 'Target' /* Milestone */ + || $word == 'Ever' /* Confirmed */ + || $word == 'Crash' /* Signature */ + || $word == 'See' /* Also */ + || $word == 'Last' /* Resolved */) + { + if ($i + 1 >= count( $words )) { + fail( 'Unrecognized field list (4): ' . print_r( $words, true ) ); + } + $word .= ' ' . $words[ ++$i ]; + } else if ($word == 'Status') { + if ($i + 1 < count( $words ) && $words[ $i + 1 ] == 'Whiteboard') { + $word .= ' '. $words[ ++$i ]; + } + } else if ($word == 'Comment') { + if ($i + 3 < count( $words) && $words[ $i + 2 ] == 'is' && $words[ $i + 3 ] == 'private') { + $word .= ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ]; + } } + $fields[] = $word; } return $fields; } -function stripDigits( $value ) { - return preg_replace( '/(\d){6}/', '......', $value ); -} - function parseChangeTable( $fields, $rows ) { // get widths to avoid dying on new/old values with pipe characters $columns = explode( '|', $rows[0] ); @@ -211,7 +208,7 @@ function parseChangeTable( $fields, $rows ) { continue; } $matchedStart = false; - if (strpos( $fields[ $ixField ], stripDigits( $col1 ) ) === 0) { + if (strpos( $fields[ $ixField ], $col1 ) === 0) { $matchedStart = true; if ($ixField > 0) { $oldvals[] = trim( $oldval ); @@ -220,7 +217,7 @@ function parseChangeTable( $fields, $rows ) { $oldval = $col2; $newval = $col3; } - if (strpos( $fields[ $ixField ], stripDigits( $col1 ) ) === strlen( $fields[ $ixField ] ) - strlen( $col1 )) { + if (strpos( $fields[ $ixField ], $col1 ) === strlen( $fields[ $ixField ] ) - strlen( $col1 )) { if (! $matchedStart) { $oldval .= $col2; $newval .= $col3; @@ -249,7 +246,7 @@ function insertChanges( $bug, $date, $reason, &$fields, &$oldvals, &$newvals ) { } } -function saveChanges( $bug, $date, $reason, &$mailString, $allowNoChangeTable ) { +function saveChanges( $bug, $date, $reason, &$mailString ) { $ret = 0; $fields = normalizeFieldList( getField( 'changed-fields' ) ); if (count( $fields ) == 0) { @@ -264,9 +261,6 @@ function saveChanges( $bug, $date, $reason, &$mailString, $allowNoChangeTable ) if (count( $fields ) == 1 && strpos( $fields[0], " is private" ) !== FALSE) { return $ret; } - if ($allowNoChangeTable) { - return $ret; - } fail( 'No change table' ); } $tableRows = $matches[1][0][0]; @@ -447,12 +441,12 @@ function updateMetadata( $date ) { $title = trim( $matches[1] ); $author = getField( 'who' ); $matches = array(); - if (preg_match( "/Bug ID: .*?\n\n(.*\n\n)?-- \n/s", $mailString, $matches ) == 0) { + if (preg_match( "/Bug #: .*?\n\n\n(.*\n\n)?-- \n/s", $mailString, $matches ) == 0) { fail( 'No description' ); } $desc = trim( $matches[1] ); - $extracted = saveChanges( $bug, $date, $reason, $desc, true ); + $extracted = saveChanges( $bug, $date, $reason, $desc ); $desc = trim( substr( $desc, $extracted ) ); $stmt = prepare( 'INSERT INTO newbugs (bug, stamp, reason, title, author, description) VALUES (?, ?, ?, ?, ?, ?)' ); @@ -465,7 +459,7 @@ function updateMetadata( $date ) { } else if ($type == 'changed') { $reason = normalizeReason( getField( 'reason' ), getField( 'watch-reason' ) ); - if (saveChanges( $bug, $date, $reason, $mailString, false ) == 0) { + if (saveChanges( $bug, $date, $reason, $mailString ) == 0) { saveDependencyChanges( $bug, $date, $reason, $mailString ); } saveComments( $bug, $date, $reason, $mailString ); From 1b5079bac505a118f058865ea473001652cb17f3 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 24 Aug 2012 08:47:47 -0400 Subject: [PATCH 013/141] Update to deal with minor bugmail format changes --- filter.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/filter.php b/filter.php index f03f83e..84771d4 100644 --- a/filter.php +++ b/filter.php @@ -186,6 +186,13 @@ function normalizeFieldList( $fieldString ) { $word .= ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ]; } } + if ($word == 'Ever Confirmed') { + $word = 'Ever confirmed'; + } else if ($word == 'OS/Version') { + $word = 'OS'; + } else if ($word == 'Platform') { + $word = 'Hardware'; + } $fields[] = $word; } return $fields; @@ -307,7 +314,7 @@ function saveDependencyChanges( $bug, $date, $reason, &$mailString ) { // in this case we don't know the list of field names ahead of time, just that it will be one or more Status // and Resolution fields. Since these won't line wrap, we can just do a simpler version of parseChangeTable $matches = array(); - if (preg_match( "/\n *What *\|Old Value *\|New Value\n-*\n(.*?)\n\n/s", $mailString, $matches, PREG_OFFSET_CAPTURE, $offset ) == 0) { + if (preg_match( "/\n *What *\|Removed *\|Added\n-*\n(.*?)\n\n/s", $mailString, $matches, PREG_OFFSET_CAPTURE, $offset ) == 0) { fail( 'Did not find change table corresponding to dependency change for bug ' . $dependentBug ); } $tableRows = explode( "\n", $matches[1][0] ); From 3fae6f586b1a7cb74023373adc56b097d71a6f01 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 24 Aug 2012 10:08:10 -0400 Subject: [PATCH 014/141] Deal with some more minor format updates --- filter.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/filter.php b/filter.php index 84771d4..ba300b4 100644 --- a/filter.php +++ b/filter.php @@ -192,6 +192,8 @@ function normalizeFieldList( $fieldString ) { $word = 'OS'; } else if ($word == 'Platform') { $word = 'Hardware'; + } else if ($word == 'AssignedTo') { + $word = 'Assignee'; } $fields[] = $word; } @@ -448,7 +450,7 @@ function updateMetadata( $date ) { $title = trim( $matches[1] ); $author = getField( 'who' ); $matches = array(); - if (preg_match( "/Bug #: .*?\n\n\n(.*\n\n)?-- \n/s", $mailString, $matches ) == 0) { + if (preg_match( "/Bug ID: .*?\n\n(.*\n\n)?-- \n/s", $mailString, $matches ) == 0) { fail( 'No description' ); } $desc = trim( $matches[1] ); From 89ae8203a0df7a5231c463c0c115d798fbab1f8e Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 24 Aug 2012 15:42:38 -0400 Subject: [PATCH 015/141] Update whiteboard field name --- filter.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/filter.php b/filter.php index ba300b4..84faa8e 100644 --- a/filter.php +++ b/filter.php @@ -194,6 +194,8 @@ function normalizeFieldList( $fieldString ) { $word = 'Hardware'; } else if ($word == 'AssignedTo') { $word = 'Assignee'; + } else if ($word == 'Status Whiteboard') { + $word = 'Whiteboard'; } $fields[] = $word; } From fa68a8a4397bc9223730b4c8a915c6cec1d467bb Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sat, 25 Aug 2012 10:37:07 -0400 Subject: [PATCH 016/141] The headers don't contain the attachment number any more; update to account for that --- filter.php | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/filter.php b/filter.php index 84faa8e..3ebdd2c 100644 --- a/filter.php +++ b/filter.php @@ -144,7 +144,7 @@ function normalizeFieldList( $fieldString ) { $currentField = ''; for ($i = 0; $i < count( $words ); $i++) { $word = $words[ $i ]; - if ($word == 'Attachment' /* Created|#abcdef */) { + if ($word == 'Attachment' /* Created|Flags|is|mime */) { if ($i + 1 >= count( $words )) { fail( 'Unrecognized field list (1): ' . print_r( $words, true ) ); } @@ -153,18 +153,11 @@ function normalizeFieldList( $fieldString ) { // ignore "Attachment Created" in the field list since it doesn't have // a corresponding entry in the field table continue; - } else { - /* Flags|is|mime */ + } else if ($words[ $i ] == 'is' /* obsolete */ || $words[ $i ] == 'mime' /* type */) { if ($i + 1 >= count( $words )) { - fail( 'Unrecognized field list (2): ' . print_r( $words, true ) ); + fail( 'Unrecognized field list (3): ' . print_r( $words, true ) ); } $word .= ' ' . $words[ ++$i ]; - if ($words[ $i ] == 'is' /* obsolete */ || $words[ $i ] == 'mime' /* type */) { - if ($i + 1 >= count( $words )) { - fail( 'Unrecognized field list (3): ' . print_r( $words, true ) ); - } - $word .= ' ' . $words[ ++$i ]; - } } } else if ($word == 'Depends' /* On */ || $word == 'Target' /* Milestone */ @@ -196,12 +189,20 @@ function normalizeFieldList( $fieldString ) { $word = 'Assignee'; } else if ($word == 'Status Whiteboard') { $word = 'Whiteboard'; + } else if ($word == 'QAContact') { + $word = 'QA Contact'; + } else if ($word == 'Flags') { + $word = 'Attachment Flags'; } $fields[] = $word; } return $fields; } +function stripAttachmentNumber( $value ) { + return preg_replace( '/ #\d+/', '', $value ); +} + function parseChangeTable( $fields, $rows ) { // get widths to avoid dying on new/old values with pipe characters $columns = explode( '|', $rows[0] ); @@ -210,7 +211,7 @@ function parseChangeTable( $fields, $rows ) { $newval = ''; $ixField = 0; for ($i = 1; $i < count( $rows ); $i++) { - $col1 = trim( substr( $rows[$i], 0, $widths[0] ) ); + $col1 = trim( stripAttachmentNumber( substr( $rows[$i], 0, $widths[0] ) ) ); $col2 = substr( $rows[$i], $widths[0] + 1, $widths[1] ); $col3 = substr( $rows[$i], $widths[0] + 1 + $widths[1] + 1 ); if (strlen( $col1 ) == 0 || $ixField >= count( $fields )) { From d7049a251d61710180fbc8c17059b6117de97293 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 30 Aug 2012 10:16:00 -0400 Subject: [PATCH 017/141] Allow missing change table for new bug notifications, since now we're getting changed-field headers for them too --- filter.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/filter.php b/filter.php index 3ebdd2c..fef69cf 100644 --- a/filter.php +++ b/filter.php @@ -258,7 +258,7 @@ function insertChanges( $bug, $date, $reason, &$fields, &$oldvals, &$newvals ) { } } -function saveChanges( $bug, $date, $reason, &$mailString ) { +function saveChanges( $bug, $date, $reason, &$mailString, $requireTable ) { $ret = 0; $fields = normalizeFieldList( getField( 'changed-fields' ) ); if (count( $fields ) == 0) { @@ -273,7 +273,11 @@ function saveChanges( $bug, $date, $reason, &$mailString ) { if (count( $fields ) == 1 && strpos( $fields[0], " is private" ) !== FALSE) { return $ret; } - fail( 'No change table' ); + if ($requireTable) { + fail( 'No change table' ); + } else { + return $ret; + } } $tableRows = $matches[1][0][0]; $ret = max( $ret, $matches[0][0][1] + strlen( $matches[0][0][0] ) ); @@ -458,7 +462,7 @@ function updateMetadata( $date ) { } $desc = trim( $matches[1] ); - $extracted = saveChanges( $bug, $date, $reason, $desc ); + $extracted = saveChanges( $bug, $date, $reason, $desc, false ); $desc = trim( substr( $desc, $extracted ) ); $stmt = prepare( 'INSERT INTO newbugs (bug, stamp, reason, title, author, description) VALUES (?, ?, ?, ?, ?, ?)' ); @@ -471,7 +475,7 @@ function updateMetadata( $date ) { } else if ($type == 'changed') { $reason = normalizeReason( getField( 'reason' ), getField( 'watch-reason' ) ); - if (saveChanges( $bug, $date, $reason, $mailString ) == 0) { + if (saveChanges( $bug, $date, $reason, $mailString, true ) == 0) { saveDependencyChanges( $bug, $date, $reason, $mailString ); } saveComments( $bug, $date, $reason, $mailString ); From e106776e87bb47d90da43c68a1fe02768564776c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 3 Sep 2012 18:11:42 -0400 Subject: [PATCH 018/141] Deal with flags and attachment flags having the same header --- filter.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/filter.php b/filter.php index fef69cf..4ed9766 100644 --- a/filter.php +++ b/filter.php @@ -191,8 +191,6 @@ function normalizeFieldList( $fieldString ) { $word = 'Whiteboard'; } else if ($word == 'QAContact') { $word = 'QA Contact'; - } else if ($word == 'Flags') { - $word = 'Attachment Flags'; } $fields[] = $word; } @@ -200,7 +198,7 @@ function normalizeFieldList( $fieldString ) { } function stripAttachmentNumber( $value ) { - return preg_replace( '/ #\d+/', '', $value ); + return preg_replace( '/Attachment #\d+/', '', $value ); } function parseChangeTable( $fields, $rows ) { From 7e0242ee32e5393581ef562cdf3b7299823f265a Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 4 Sep 2012 19:50:21 -0400 Subject: [PATCH 019/141] Deal with attachment flags better --- filter.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/filter.php b/filter.php index 4ed9766..0fa64e4 100644 --- a/filter.php +++ b/filter.php @@ -197,10 +197,6 @@ function normalizeFieldList( $fieldString ) { return $fields; } -function stripAttachmentNumber( $value ) { - return preg_replace( '/Attachment #\d+/', '', $value ); -} - function parseChangeTable( $fields, $rows ) { // get widths to avoid dying on new/old values with pipe characters $columns = explode( '|', $rows[0] ); @@ -209,7 +205,7 @@ function parseChangeTable( $fields, $rows ) { $newval = ''; $ixField = 0; for ($i = 1; $i < count( $rows ); $i++) { - $col1 = trim( stripAttachmentNumber( substr( $rows[$i], 0, $widths[0] ) ) ); + $col1 = trim( substr( $rows[$i], 0, $widths[0] ) ); $col2 = substr( $rows[$i], $widths[0] + 1, $widths[1] ); $col3 = substr( $rows[$i], $widths[0] + 1 + $widths[1] + 1 ); if (strlen( $col1 ) == 0 || $ixField >= count( $fields )) { @@ -218,7 +214,7 @@ function parseChangeTable( $fields, $rows ) { continue; } $matchedStart = false; - if (strpos( $fields[ $ixField ], $col1 ) === 0) { + if (strpos( $fields[ $ixField ], $col1 ) === 0 || ($fields[ $ixField ] == 'Flags' && preg_match( '/Attachment #\d+/', $col1 ))) { $matchedStart = true; if ($ixField > 0) { $oldvals[] = trim( $oldval ); From 8d32103941dec204b037ae7f648e7caa5fbb3998 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 5 Sep 2012 09:40:55 -0400 Subject: [PATCH 020/141] Really fix the attachment flags thing this time --- filter.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/filter.php b/filter.php index 0fa64e4..7beda35 100644 --- a/filter.php +++ b/filter.php @@ -214,8 +214,13 @@ function parseChangeTable( $fields, $rows ) { continue; } $matchedStart = false; - if (strpos( $fields[ $ixField ], $col1 ) === 0 || ($fields[ $ixField ] == 'Flags' && preg_match( '/Attachment #\d+/', $col1 ))) { + if (strpos( $fields[ $ixField ], $col1 ) === 0) { $matchedStart = true; + } else if ($fields[ $ixField ] == 'Flags' && preg_match( '/Attachment #\d+/', $col1 )) { + $fields[ $ixField ] = $col1 . ' ' . $fields[ $ixField ]; + $matchedStart = true; + } + if ($matchedStart) { if ($ixField > 0) { $oldvals[] = trim( $oldval ); $newvals[] = trim( $newval ); From 378d90bbb89e9be790dc380394400b1f9689ad0e Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 5 Sep 2012 11:17:08 -0400 Subject: [PATCH 021/141] Update _ME to be an array --- README.html | 2 +- dashboard.php | 4 +++- example.config.php | 4 ++-- filter.php | 2 +- tags.php | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.html b/README.html index 41beec1..7555f76 100644 --- a/README.html +++ b/README.html @@ -32,7 +32,7 @@

Detailed Setup

"| /home/myusername/path/to/decrypt_mail.awk | /home/myusername/path/to/filter.php" -

The second component is a MySQL database. This is where all the bug data is actually stored after being extracted from the emails. You will need to set up a database with the necessary tables (schemas are in tables.sql). You will also need to rename the example.config.php file to bugmash.config.php and put your database access information into it. While you're at it, also fill in $_ME in the config file with the email address on your Bugzilla account (i.e. what shows up when you go to Bugzilla preferences).

+

The second component is a MySQL database. This is where all the bug data is actually stored after being extracted from the emails. You will need to set up a database with the necessary tables (schemas are in tables.sql). You will also need to rename the example.config.php file to bugmash.config.php and put your database access information into it. While you're at it, also fill in $_ME in the config file with the email address on your Bugzilla account (i.e. what shows up when you go to Bugzilla preferences). If you have multiple Bugzilla accounts you can put them all in.

IMPORTANT! Make sure that the bugmash.config.php file is not in your web server's document root (generally $HOME/www), otherwise your database credentials may be at risk of being snarfed by evil crackers!!

diff --git a/dashboard.php b/dashboard.php index 4f7a252..a6cf6f2 100644 --- a/dashboard.php +++ b/dashboard.php @@ -178,7 +178,9 @@ function column( &$reasons ) { ($row['cancelled'] ? '' : '') ) . "\n"; $reasons[ $row['bug'] ][] = 'request'; - $filterFlags[ $row['attachment'] ][] = array( '', "{$row['flag']}?({$_ME})" ); + foreach ($_ME as $myEmail) { + $filterFlags[ $row['attachment'] ][] = array( '', "{$row['flag']}?({$myEmail})" ); + } } $result = loadTable( 'newbugs' ); diff --git a/example.config.php b/example.config.php index 66315e0..04bb106 100644 --- a/example.config.php +++ b/example.config.php @@ -9,8 +9,8 @@ $_MYSQL_PASS = 'bugdb_pass' $_MYSQL_DB = 'bugmash'; -// My bugmail address -$_ME = 'bugmail@bugdb.example.com'; +// My bugmail addresses +$_ME = array( 'bugmail@bugdb.example.com' ); // The extension on the final email address that is receiving the email. You can always leave this unset to disable extension checking. // $_MY_EXTENSION = 'bugmash'; diff --git a/filter.php b/filter.php index 7beda35..296553a 100644 --- a/filter.php +++ b/filter.php @@ -431,7 +431,7 @@ function updateMetadata( $date ) { } } else { $requestee = getField( 'flag-requestee' ); - if ($requestee != $_ME) { + if (! in_array( $requestee, $_ME )) { fail( 'Requestee is not me' ); } $flag = substr( $subject, 0, strpos( $subject, ' ' ) ); diff --git a/tags.php b/tags.php index 6094df8..1a9029e 100644 --- a/tags.php +++ b/tags.php @@ -2,7 +2,7 @@ include_once( 'common.php' ); -if (! (isset( $_POST['user'] ) && $_POST['user'] == $_ME)) { +if (! (isset( $_POST['user'] ) && in_array( $_POST['user'], $_ME ))) { fail( 'Incorrect user: ' . $_POST['user'] ); } From 0b444ffdd7b4a6fca528552280b60fafc3e90475 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 5 Sep 2012 15:56:29 -0400 Subject: [PATCH 022/141] Update filter to deal with KDE bugmail as well (field headers are lowercase, the from address is bugzilla_noreply) --- filter.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/filter.php b/filter.php index 296553a..8965f03 100644 --- a/filter.php +++ b/filter.php @@ -20,7 +20,7 @@ } } -if ((! isset( $_SERVER['SENDER'] )) || (strpos( $_SERVER['SENDER'], 'bugzilla-daemon@' ) !== 0)) { +if ((! isset( $_SERVER['SENDER'] )) || (strpos( $_SERVER['SENDER'], 'bugzilla' ) !== 0)) { // doesn't look like a bugmail, probably spam but possible bounce notifications. put it aside file_put_contents( $_UNFILTERED_DIR . '/' . $filename, $mailString ); exit( 0 ); @@ -214,9 +214,9 @@ function parseChangeTable( $fields, $rows ) { continue; } $matchedStart = false; - if (strpos( $fields[ $ixField ], $col1 ) === 0) { + if (stripos( $fields[ $ixField ], $col1 ) === 0) { $matchedStart = true; - } else if ($fields[ $ixField ] == 'Flags' && preg_match( '/Attachment #\d+/', $col1 )) { + } else if ($fields[ $ixField ] == 'Flags' && preg_match( '/Attachment #\d+/i', $col1 )) { $fields[ $ixField ] = $col1 . ' ' . $fields[ $ixField ]; $matchedStart = true; } @@ -228,7 +228,7 @@ function parseChangeTable( $fields, $rows ) { $oldval = $col2; $newval = $col3; } - if (strpos( $fields[ $ixField ], $col1 ) === strlen( $fields[ $ixField ] ) - strlen( $col1 )) { + if (stripos( $fields[ $ixField ], $col1 ) === strlen( $fields[ $ixField ] ) - strlen( $col1 )) { if (! $matchedStart) { $oldval .= $col2; $newval .= $col3; From b5553e0141f2c0766211e6a4248cea5abcdfaa57 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 7 Sep 2012 09:47:07 -0400 Subject: [PATCH 023/141] Deal with mozilla bug 789278 --- bugtags.user.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bugtags.user.js b/bugtags.user.js index 2fd7663..5bf034c 100644 --- a/bugtags.user.js +++ b/bugtags.user.js @@ -89,7 +89,7 @@ function insertListBugTags( user, bugnumbers ) { url: TAGS_SERVER, data: reqData, onload: function( res ) { - var response = res.responseJSON; + var response = JSON.parse( res.responseText ); var rows = document.getElementsByClassName( "bz_buglist" )[0].getElementsByClassName( "bz_bugitem" ); for (var i = 0; i < rows.length; i++) { var row = rows[i]; @@ -118,7 +118,7 @@ function insertTreeBugTags( user, bugnumbers ) { url: TAGS_SERVER, data: reqData, onload: function( res ) { - var response = res.responseJSON; + var response = JSON.parse( res.responseText ); var nodes = document.getElementsByClassName( "tree" )[0].getElementsByClassName( "summ_deep" ); for (var i = 0; i < nodes.length; i++) { var node = nodes[i]; From 071329be75fb8459592b42f4d0778dff11dd571f Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 10 Sep 2012 09:51:35 -0400 Subject: [PATCH 024/141] Deal with other attachment fields like mime and plain --- filter.php | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/filter.php b/filter.php index 8965f03..b5c883f 100644 --- a/filter.php +++ b/filter.php @@ -216,9 +216,13 @@ function parseChangeTable( $fields, $rows ) { $matchedStart = false; if (stripos( $fields[ $ixField ], $col1 ) === 0) { $matchedStart = true; - } else if ($fields[ $ixField ] == 'Flags' && preg_match( '/Attachment #\d+/i', $col1 )) { - $fields[ $ixField ] = $col1 . ' ' . $fields[ $ixField ]; - $matchedStart = true; + } else if (preg_match( '/Attachment #\d+/i', $col1 )) { + if ($fields[ $ixField ] == 'Flags') { + $fields[ $ixField ] = $col1 . ' ' . $fields[ $ixField ]; + $matchedStart = true; + } else if (stripos( $fields[ $ixField ], preg_replace( '/Attachment #\d+/i', 'Attachment', $col1 ) ) === 0) { + $matchedStart = true; + } } if ($matchedStart) { if ($ixField > 0) { From 250c04743ab20a665a8312a117bc70b01482b28e Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sun, 23 Sep 2012 13:45:33 -0400 Subject: [PATCH 025/141] Add a X at the bottom of the bug block too --- dashboard.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dashboard.php b/dashboard.php index a6cf6f2..f1810f2 100644 --- a/dashboard.php +++ b/dashboard.php @@ -270,7 +270,10 @@ function column( &$reasons ) { . 'Bug %d %s' . '' . '
%s
' - . '
Back to top
' + . '
' + . 'X' + . 'Back to top' + . '
' . '', ($meta_secure[ $bug ] ? 'secure ' : ''), $bug, @@ -282,6 +285,7 @@ function column( &$reasons ) { $bug, escapeHTML( $meta_titles[ $bug ] ), implode( "\n", $block ), + $bug, $bug ) . "\n"; $columns[ column( $reasons[ $bug ] ) ][ $touchTime ] .= $block; } @@ -352,7 +356,7 @@ function column( &$reasons ) { background-color: red; color: white; } -a.wipe { +a.wipe, a.wipetop { float: right; margin-left: 3px; vertical-align: top; From 0cb1791efc4d1ab2a774afa7b24a7181887c7da8 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 29 Oct 2012 11:17:04 -0400 Subject: [PATCH 026/141] Deal with missing email addresses in unconfirmed bugmails --- filter.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/filter.php b/filter.php index b5c883f..c9f878b 100644 --- a/filter.php +++ b/filter.php @@ -297,11 +297,15 @@ function saveChanges( $bug, $date, $reason, &$mailString, $requireTable ) { function saveComments( $bug, $date, $reason, &$mailString ) { $matches = array(); - $matchCount = preg_match_all( "/- Comment #(\d+) from ([^<]*) <[^\n]* ---\n(.*)\n\n--/sU", $mailString, $matches, PREG_PATTERN_ORDER ); + $matchCount = preg_match_all( "/- Comment #(\d+) from ([^\n]*) ---\n(.*)\n\n--/sU", $mailString, $matches, PREG_PATTERN_ORDER ); $stmt = prepare( 'INSERT INTO comments (bug, stamp, reason, commentnum, author, comment) VALUES (?, ?, ?, ?, ?, ?)' ); for ($i = 0; $i < $matchCount; $i++) { $commentNum = $matches[1][$i]; $author = $matches[2][$i]; + $author = substr( $author, 0, 0 - strlen( 'YYYY-mm-dd HH:ii::ss ZZZ' ) ); + if (strpos( $author, '<' ) !== FALSE) { + $author = substr( $author, 0, strpos( $author, '<' ) ); + } $comment = $matches[3][$i]; $stmt->bind_param( 'ississ', $bug, $date, $reason, $commentNum, $author, $comment ); $stmt->execute(); From b9044214674f43a0953a398d0f2470917102dc0a Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 3 Jan 2013 10:21:05 -0500 Subject: [PATCH 027/141] Fuzzify the TBPL robot check --- dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard.php b/dashboard.php index f1810f2..12bbf98 100644 --- a/dashboard.php +++ b/dashboard.php @@ -248,7 +248,7 @@ function column( &$reasons ) { $numRows++; $stamp = strtotime( $row['stamp'] ); - $isTbplRobot = ($row['author'] == 'TinderboxPushlog Robot'); + $isTbplRobot = (stripWhitespace( $row['author'] ) == 'TinderboxPushlogRobot'); $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s said:
%s
', ($hide ? 'display: none;' : 'white-space: pre-line;'), ($isTbplRobot ? 'opacity: 0.5;' : ''), From c9adf31e29794d4b52158bbcda7d954d4dc687e5 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 6 Feb 2013 14:17:57 -0500 Subject: [PATCH 028/141] Deal with needinfo requests as well --- filter.php | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/filter.php b/filter.php index c9f878b..dd93235 100644 --- a/filter.php +++ b/filter.php @@ -390,19 +390,25 @@ function updateMetadata( $date ) { insertChanges( $bug, $date, $reason, $fields, $oldvals, $newvals ); success(); } else if ($type == 'request') { - $matches = array(); + $subject = getField( 'subject' ); + $matches = array(); if (preg_match( '/\[Attachment (\d+)\]/', $mailText, $matches ) == 0) { - fail( 'No attachment id' ); - } - $attachment = $matches[1]; - - if (preg_match( "/Attachment $attachment: (.*)/", $mailString, $matches ) == 0) { - fail( 'No attachment title' ); + if (strpos( $subject, 'needinfo requested: [Bug' ) === FALSE) { + fail( 'Unrecognized request bugmail' ); + } + // it's a needinfo request, hack the fields to insert ourself as a requestee + $bugzillaHeaders['flag-requestee'] = $_ME[0]; + $attachment = 0; + $title = ''; + } else { + $attachment = $matches[1]; + if (preg_match( "/Attachment $attachment: (.*)/", $mailString, $matches ) == 0) { + fail( 'No attachment title' ); + } + $title = $matches[1]; } - $title = $matches[1]; - $subject = getField( 'subject' ); if (! checkForField( 'flag-requestee' )) { if (strpos( $subject, ' canceled: [Bug' ) !== FALSE) { $granted = 0; From ae188c8b07f7dce555224760c1b4260f4099095a Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 6 Feb 2013 14:24:21 -0500 Subject: [PATCH 029/141] Fix longstanding bug that all requests show up as r? instead of with the correct abbreviation --- dashboard.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dashboard.php b/dashboard.php index 12bbf98..1586c0e 100644 --- a/dashboard.php +++ b/dashboard.php @@ -168,9 +168,10 @@ function column( &$reasons ) { while ($row = $result->fetch_assoc()) { $numRows++; $stamp = strtotime( $row['stamp'] ); - $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%sr? %s%s
', + $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s%s? %s%s
', $row['id'], ($row['cancelled'] ? '' : ''), + abbrevFlag( $row['flag'] ), $_BASE_URL, $row['bug'], $row['attachment'], From 5961a0d21b3722e51840e75c62cc01ab9adc98b4 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 7 Feb 2013 09:57:12 -0500 Subject: [PATCH 030/141] Handle needinfo cancellations --- filter.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/filter.php b/filter.php index dd93235..7830373 100644 --- a/filter.php +++ b/filter.php @@ -394,11 +394,12 @@ function updateMetadata( $date ) { $matches = array(); if (preg_match( '/\[Attachment (\d+)\]/', $mailText, $matches ) == 0) { - if (strpos( $subject, 'needinfo requested: [Bug' ) === FALSE) { + if (strpos( $subject, 'needinfo ' ) === FALSE) { fail( 'Unrecognized request bugmail' ); } - // it's a needinfo request, hack the fields to insert ourself as a requestee - $bugzillaHeaders['flag-requestee'] = $_ME[0]; + if (strpos( $subject, 'needinfo requested: [Bug' ) !== FALSE) { + $bugzillaHeaders['flag-requestee'] = $_ME[0]; + } $attachment = 0; $title = ''; } else { @@ -425,8 +426,8 @@ function updateMetadata( $date ) { $flag = substr( $subject, 0, strpos( $subject, ' ' ) ); if ($cancelled) { - $stmt = prepare( 'UPDATE requests SET cancelled=? WHERE attachment=? AND flag=?' ); - $stmt->bind_param( 'iis', $cancelled, $attachment, $flag ); + $stmt = prepare( 'UPDATE requests SET cancelled=? WHERE bug=? AND attachment=? AND flag=?' ); + $stmt->bind_param( 'iiis', $cancelled, $bug, $attachment, $flag ); $stmt->execute(); // this may cancel something we don't have a record of; if so, ignore success(); From fe06f5a10a8ae8659d3917dcbf43ab33669b1af8 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 28 May 2013 14:52:11 -0400 Subject: [PATCH 031/141] Add an extra space next to the X for mac os lion scrollbars --- dashboard.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard.php b/dashboard.php index 1586c0e..21efcc3 100644 --- a/dashboard.php +++ b/dashboard.php @@ -266,13 +266,13 @@ function column( &$reasons ) { ksort( $block, SORT_NUMERIC ); $touchTime = key( $block ); $block = sprintf( '
' - . 'X' + . 'X ' . '%s' . 'Bug %d %s' . '
' . '
%s
' . '
' - . 'X' + . 'X ' . 'Back to top' . '
' . '
', From 831837962fc4d19b00629fd7b8afdf0b2948a5d7 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 1 Aug 2013 11:10:36 -0400 Subject: [PATCH 032/141] Add support for dep_changed emails --- filter.php | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/filter.php b/filter.php index 7830373..64f65b3 100644 --- a/filter.php +++ b/filter.php @@ -313,6 +313,7 @@ function saveComments( $bug, $date, $reason, &$mailString ) { fail( 'Unable to insert new comment into DB: ' . $stmt->error ); } } + return $matchCount; } function saveDependencyChanges( $bug, $date, $reason, &$mailString ) { @@ -486,13 +487,20 @@ function updateMetadata( $date ) { fail( 'Unable to insert new bug into DB: ' . $stmt->error ); } success(); +} else if ($type == 'dep_changed') { + $reason = normalizeReason( getField( 'reason' ), getField( 'watch-reason' ) ); + if (saveDependencyChanges( $bug, $date, $reason, $mailString )) { + success(); + } + fail( 'Unable to parse dep_changed email' ); } else if ($type == 'changed') { $reason = normalizeReason( getField( 'reason' ), getField( 'watch-reason' ) ); - if (saveChanges( $bug, $date, $reason, $mailString, true ) == 0) { - saveDependencyChanges( $bug, $date, $reason, $mailString ); + $extracted = saveChanges( $bug, $date, $reason, $mailString, true ); + $comments = saveComments( $bug, $date, $reason, $mailString ); + if ($extracted == 0 && $comments == 0) { + fail( 'Unable to extract meaningful data from changed email' ); } - saveComments( $bug, $date, $reason, $mailString ); success(); } else { From d95c92cc4be786fb0acc82bca929a8aa9c09e05d Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 22 Aug 2013 09:10:44 -0400 Subject: [PATCH 033/141] Update robot name change --- dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard.php b/dashboard.php index 21efcc3..3434f5c 100644 --- a/dashboard.php +++ b/dashboard.php @@ -249,7 +249,7 @@ function column( &$reasons ) { $numRows++; $stamp = strtotime( $row['stamp'] ); - $isTbplRobot = (stripWhitespace( $row['author'] ) == 'TinderboxPushlogRobot'); + $isTbplRobot = (stripWhitespace( $row['author'] ) == 'TBPLRobot'); $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s said:
%s
', ($hide ? 'display: none;' : 'white-space: pre-line;'), ($isTbplRobot ? 'opacity: 0.5;' : ''), From c71846f5be706f3b74809e9cd432424658b521ef Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 18 Nov 2013 10:41:43 -0500 Subject: [PATCH 034/141] Update attachment flags parsing bit to deal with the attachment number falling onto the next line. Also makes it a bit more generic --- filter.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/filter.php b/filter.php index 64f65b3..d47f788 100644 --- a/filter.php +++ b/filter.php @@ -215,12 +215,18 @@ function parseChangeTable( $fields, $rows ) { } $matchedStart = false; if (stripos( $fields[ $ixField ], $col1 ) === 0) { + // simple case match the start of the field against the column $matchedStart = true; - } else if (preg_match( '/Attachment #\d+/i', $col1 )) { - if ($fields[ $ixField ] == 'Flags') { - $fields[ $ixField ] = $col1 . ' ' . $fields[ $ixField ]; + } else if (preg_match( '/^Attachment/', $col1 )) { + // Sometimes the attachment number is on the second line. Make sure we pick it up + $col1 .= ' ' . trim( substr( $rows[$i+1], 0, $widths[0] ) ); + // Special case for flags, because the changed header just has "Flags" instead of "Attachment Flags" + if ($fields[ $ixField ] == 'Flags' && preg_match( '/^Attachment #\d+ Flags/', $col1 )) { + $fields[ $ixField ] = $col1; $matchedStart = true; + // Otherwise insert the attachment number into $fields[$ixField] } else if (stripos( $fields[ $ixField ], preg_replace( '/Attachment #\d+/i', 'Attachment', $col1 ) ) === 0) { + $fields[ $ixField ] = preg_replace( '/(Attachment #\d+).*/', '\1', $col1 ) . substr( $fields[ $ixField ], strlen( 'Attachment' )); $matchedStart = true; } } From dd66e4ddae2a213ad6d2a7d37b0be555568a82bf Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 10 Jan 2014 10:20:30 -0500 Subject: [PATCH 035/141] Deal with linebreaks when people with long names respond to review requests --- filter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.php b/filter.php index d47f788..a00af1d 100644 --- a/filter.php +++ b/filter.php @@ -439,7 +439,7 @@ function updateMetadata( $date ) { // this may cancel something we don't have a record of; if so, ignore success(); } else { - if (preg_match( "/\n(.*) <(.*)> has (?:not )?granted/", $mailString, $matches ) == 0) { + if (preg_match( "/\n(.*) <(.*)>\\shas\\s(?:not\\s)?granted/", $mailString, $matches ) == 0) { fail( 'Unable to determine author of review' ); } $author = $matches[1]; From 45aac82716b0d31aa0eee667b0631ca44388f943 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 6 Mar 2014 11:32:05 -0500 Subject: [PATCH 036/141] Add User Story as a field --- filter.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/filter.php b/filter.php index a00af1d..85aba94 100644 --- a/filter.php +++ b/filter.php @@ -172,12 +172,16 @@ function normalizeFieldList( $fieldString ) { $word .= ' ' . $words[ ++$i ]; } else if ($word == 'Status') { if ($i + 1 < count( $words ) && $words[ $i + 1 ] == 'Whiteboard') { - $word .= ' '. $words[ ++$i ]; + $word .= ' ' . $words[ ++$i ]; } } else if ($word == 'Comment') { if ($i + 3 < count( $words) && $words[ $i + 2 ] == 'is' && $words[ $i + 3 ] == 'private') { $word .= ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ]; } + } else if ($word == 'User') { + if ($i + 1 < count( $words ) && $words[ $i + 1 ] == 'Story') { + $word .= ' ' . $words[ ++$i ]; + } } if ($word == 'Ever Confirmed') { $word = 'Ever confirmed'; From 30df22a830bb1362951782f3e4915cc27a70fa57 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 30 Apr 2014 09:23:16 -0400 Subject: [PATCH 037/141] Add QA Whiteboard field --- filter.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/filter.php b/filter.php index 85aba94..e724e93 100644 --- a/filter.php +++ b/filter.php @@ -164,7 +164,8 @@ function normalizeFieldList( $fieldString ) { || $word == 'Ever' /* Confirmed */ || $word == 'Crash' /* Signature */ || $word == 'See' /* Also */ - || $word == 'Last' /* Resolved */) + || $word == 'Last' /* Resolved */ + || $word == 'QA' /* Whiteboard */) { if ($i + 1 >= count( $words )) { fail( 'Unrecognized field list (4): ' . print_r( $words, true ) ); From 37c66ef913be07610f21e698d6073652b78b422c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 13 May 2014 11:17:55 -0400 Subject: [PATCH 038/141] Scroll to the top of a bug block on x-axis too --- dashboard.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dashboard.php b/dashboard.php index 3434f5c..f5f06c4 100644 --- a/dashboard.php +++ b/dashboard.php @@ -272,8 +272,8 @@ function column( &$reasons ) { . '' . '
%s
' . '
' - . 'X ' - . 'Back to top' + . 'X ' + . 'Back to top' . '
' . '', ($meta_secure[ $bug ] ? 'secure ' : ''), @@ -287,6 +287,8 @@ function column( &$reasons ) { escapeHTML( $meta_titles[ $bug ] ), implode( "\n", $block ), $bug, + $bug, + $bug, $bug ) . "\n"; $columns[ column( $reasons[ $bug ] ) ][ $touchTime ] .= $block; } From 6c72b2af3ffa1ccaa05d044b95c915bb383bee16 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 8 Jul 2014 09:58:19 -0400 Subject: [PATCH 039/141] Deal with new 'comment created' useless field --- filter.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/filter.php b/filter.php index e724e93..c0b73ed 100644 --- a/filter.php +++ b/filter.php @@ -175,8 +175,13 @@ function normalizeFieldList( $fieldString ) { if ($i + 1 < count( $words ) && $words[ $i + 1 ] == 'Whiteboard') { $word .= ' ' . $words[ ++$i ]; } - } else if ($word == 'Comment') { - if ($i + 3 < count( $words) && $words[ $i + 2 ] == 'is' && $words[ $i + 3 ] == 'private') { + } else if ($word == 'Comment' /* Created | # is private */) { + if ($i + 1 < count( $words ) && $words[ $i + 1 ] == 'Created') { + // ignore "Comment Created" in the field list since it doesn't have + // a corresponding entry in the field table + $i++; + continue; + } else if ($i + 3 < count( $words ) && $words[ $i + 2 ] == 'is' && $words[ $i + 3 ] == 'private') { $word .= ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ]; } } else if ($word == 'User') { From 429372eaf707143d80f065db635e87ac3654f8c2 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 8 Jul 2014 09:59:00 -0400 Subject: [PATCH 040/141] Slight edit to user script while debugging GM_xhr failures --- bugtags.user.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bugtags.user.js b/bugtags.user.js index 5bf034c..c8e4af9 100644 --- a/bugtags.user.js +++ b/bugtags.user.js @@ -7,6 +7,7 @@ // @description Allows you tag bugs; the tags are then shown on Bugzilla pages // @match https://bugzilla.mozilla.org/* // @run-at document-end +// @grant unsafeWindow // ==/UserScript== var TAGS_SERVER = 'https://example.com/path/to/tags.php'; // point this to your tags.php @@ -84,7 +85,7 @@ function insertListBugTags( user, bugnumbers ) { reqData.append( "action", "get" ); reqData.append( "bugs", bugnumbers.join( "," ) ); - GM_xmlhttpRequest({ + var arg = { method: "POST", url: TAGS_SERVER, data: reqData, @@ -104,7 +105,8 @@ function insertListBugTags( user, bugnumbers ) { GM_log( res.statusText ); GM_log( res.responseText ); } - }); + }; + GM_xmlhttpRequest(arg); } function insertTreeBugTags( user, bugnumbers ) { From d3a04f8c33434f3a0829b120c2ac5cc8c531a1dd Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 10 Dec 2014 15:05:04 -0500 Subject: [PATCH 041/141] Update filter script --- filter.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filter.php b/filter.php index c0b73ed..0fddf2c 100644 --- a/filter.php +++ b/filter.php @@ -1,4 +1,4 @@ -#!/usr/local/bin/php +#!/usr/bin/env php \\shas\\s(?:not\\s)?granted/", $mailString, $matches ) == 0) { + if (preg_match( "/\n\n(.*) <(.*)>\\shas\\s(?:not\\s)?granted/s", $mailString, $matches ) == 0) { fail( 'Unable to determine author of review' ); } $author = $matches[1]; From 8b144ec2a756cc91d8fa06955658f88c3a26c7a1 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 10 Dec 2014 15:18:06 -0500 Subject: [PATCH 042/141] Deal with new secure mail format --- filter.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/filter.php b/filter.php index 0fddf2c..f6485e2 100644 --- a/filter.php +++ b/filter.php @@ -77,12 +77,7 @@ function success() { $bugzillaHeaders = array(); foreach ($mail as $mailLine) { if (strlen( $mailLine ) == 0) { - $matches = array(); - if (preg_match( '/\[Bug \d+\] \(Secure bug/', $bugzillaHeaders[ 'subject' ], $matches ) == 0) { - break; - } - $bugIsSecure = true; - // continue processing since there will be another subject header in the body, followed by another blank line + break; } if (strpos( $mailLine, 'X-Bugzilla-' ) === 0) { $header = substr( $mailLine, strlen( 'X-Bugzilla-' ) ); @@ -103,6 +98,10 @@ function success() { } } +if (checkForField( 'Secure-Email' )) { + $bugIsSecure = true; +} + function checkForField( $key ) { global $bugzillaHeaders; $key = strtolower( $key ); @@ -398,7 +397,7 @@ function updateMetadata( $date ) { updateMetadata( $date ); -if (strpos( $mailText, 'This email would have contained sensitive information' ) !== FALSE) { +if ($bugIsSecure) { // you haven't set a PGP/GPG key and this is for a secure bug, so there's no data in it. $reason = normalizeReason( getField( 'reason' ), getField( 'watch-reason' ) ); $fields = array( 'Unknown' ); From 0ca0f0951926e146b5e44c3ea0a525f485c60b37 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sat, 27 Dec 2014 09:27:30 -0500 Subject: [PATCH 043/141] Deal with attachment titles that fall into the next line because of length --- filter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.php b/filter.php index f6485e2..1ac1430 100644 --- a/filter.php +++ b/filter.php @@ -420,7 +420,7 @@ function updateMetadata( $date ) { $title = ''; } else { $attachment = $matches[1]; - if (preg_match( "/Attachment $attachment: (.*)/", $mailString, $matches ) == 0) { + if (preg_match( "/Attachment $attachment:[ \n](.*)/", $mailString, $matches ) == 0) { fail( 'No attachment title' ); } $title = $matches[1]; From 52ad6ae5ff0d579349cb541a3fbbd736af8a9d9c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 23 Mar 2015 08:05:28 -0400 Subject: [PATCH 044/141] Add treeherder robot --- dashboard.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dashboard.php b/dashboard.php index f5f06c4..ff4aeb7 100644 --- a/dashboard.php +++ b/dashboard.php @@ -249,7 +249,8 @@ function column( &$reasons ) { $numRows++; $stamp = strtotime( $row['stamp'] ); - $isTbplRobot = (stripWhitespace( $row['author'] ) == 'TBPLRobot'); + $isTbplRobot = (stripWhitespace( $row['author'] ) == 'TBPLRobot') + || (stripWhitespace( $row['author'] ) == 'TreeherderRobot'); $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s said:
%s
', ($hide ? 'display: none;' : 'white-space: pre-line;'), ($isTbplRobot ? 'opacity: 0.5;' : ''), From 38987029f345a508aa28e1401a4e118519f60b17 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 17 Apr 2015 16:50:48 -0400 Subject: [PATCH 045/141] Suppress warning on amazon AMIs --- filter.php | 1 + 1 file changed, 1 insertion(+) diff --git a/filter.php b/filter.php index 1ac1430..dce4866 100644 --- a/filter.php +++ b/filter.php @@ -11,6 +11,7 @@ $time = (isset( $_SERVER['REQUEST_TIME'] ) ? $_SERVER['REQUEST_TIME'] : time()); $filename = $time . '.' . sha1( $mailString ); +date_default_timezone_set( 'UTC' ); if (isset( $_MY_EXTENSION )) { if ((! isset( $_SERVER['EXTENSION'] )) || strcmp( $_SERVER['EXTENSION'], $_MY_EXTENSION ) != 0) { From 884759bec6df4b8847962ace2ef9a7d1a2eaba5c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 17 Apr 2015 16:51:01 -0400 Subject: [PATCH 046/141] Remove spurious variable --- filter.php | 1 - 1 file changed, 1 deletion(-) diff --git a/filter.php b/filter.php index dce4866..2f624d4 100644 --- a/filter.php +++ b/filter.php @@ -74,7 +74,6 @@ function success() { $mail = $merged; $bugIsSecure = false; -$fromDomain = substr( $_SERVER['SENDER'], strpos( $_SERVER['SENDER'], '@' ) + 1 ); $bugzillaHeaders = array(); foreach ($mail as $mailLine) { if (strlen( $mailLine ) == 0) { From 5c2a9aec7f2291b10210aa0c67d4b7b2f4fd957c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 17 Apr 2015 16:53:37 -0400 Subject: [PATCH 047/141] Make sender checking optional --- example.config.php | 5 +++++ filter.php | 10 ++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/example.config.php b/example.config.php index 04bb106..ae2160d 100644 --- a/example.config.php +++ b/example.config.php @@ -15,6 +15,11 @@ // The extension on the final email address that is receiving the email. You can always leave this unset to disable extension checking. // $_MY_EXTENSION = 'bugmash'; +// Check the SENDER environment variable to see if the mail came from bugzilla. +// If legit-looking bugmail are ending up in your spam folder you might want to +// set this to 0 or comment it out. +$_CHECK_SENDER = 1; + // directory where emails that are not bugzilla emails will be dropped $_UNFILTERED_DIR = $_SERVER['HOME'] . '/Maildir/new'; diff --git a/filter.php b/filter.php index 2f624d4..a520fb7 100644 --- a/filter.php +++ b/filter.php @@ -21,10 +21,12 @@ } } -if ((! isset( $_SERVER['SENDER'] )) || (strpos( $_SERVER['SENDER'], 'bugzilla' ) !== 0)) { - // doesn't look like a bugmail, probably spam but possible bounce notifications. put it aside - file_put_contents( $_UNFILTERED_DIR . '/' . $filename, $mailString ); - exit( 0 ); +if (isset( $_CHECK_SENDER ) && intval( $_CHECK_SENDER )) { + if ((! isset( $_SERVER['SENDER'] )) || (strpos( $_SERVER['SENDER'], 'bugzilla' ) !== 0)) { + // doesn't look like a bugmail, probably spam but possible bounce notifications. put it aside + file_put_contents( $_UNFILTERED_DIR . '/' . $filename, $mailString ); + exit( 0 ); + } } function fail( $message ) { From 0ad5b89ae30a804cf968ffa164d051c2333dd3a2 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sat, 2 May 2015 10:00:22 -0400 Subject: [PATCH 048/141] Fix scrollTo onclick code --- dashboard.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard.php b/dashboard.php index ff4aeb7..9d7994e 100644 --- a/dashboard.php +++ b/dashboard.php @@ -273,8 +273,8 @@ function column( &$reasons ) { . '' . '
%s
' . '
' - . 'X ' - . 'Back to top' + . 'X ' + . 'Back to top' . '
' . '', ($meta_secure[ $bug ] ? 'secure ' : ''), From febc9a24846941b9b353f1a5a1a71a294fa22379 Mon Sep 17 00:00:00 2001 From: retornam Date: Tue, 5 May 2015 10:28:27 -0700 Subject: [PATCH 049/141] Create README.md --- README.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..a31a550 --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +# Bugmash v0.1 + +A dashboard for quickly viewing and processing high volumes of bugmail. Also comes with the ability to add private notes and tags to bugs, and have those tags show up on Bugzilla itself (via a Scriptish user script). + +* Source code available at: [https://github.com/staktrace/bugmash](https://github.com/staktrace/bugmash) +* Original author: Kartikaya Gupta +* Contributors: maybe you! + +Jump to section: [Quick Setup](#qsetup) | [Detailed Setup](#dsetup) | [Usage](#usage) | [Known Issues](#kissues) | [Areas for improvement](#improve) + +## Quick Setup + +Bugmash is NOT trivial to install. At least, not yet. It also requires that you have some infrastructure set up on your part (database, PHP, etc.) to be able to run the different pieces. Therefore, use of bugmash is not for the faint of heart, and is really only recommended if the volume of bugmail you receive (or want to receive) is stupendous. A good qualification factor might be that you have declared bugmail bankruptcy at least once. Therefore, there is no quick setup. Work your way through the detailed setup instructions if you're still interested. + +## Detailed Setup + +Bugmash has a number of components that work together. The way you set up these components can vary somewhat, but you may need to twiddle a few things here and there to get things to work if you deviate too far from how I've set it up. Ideally as more people try to use it, they will contribute patches to make this easier. As it is, you should probably read through these instructions first before even attempting to set it up. + +First, there is an email scraper. This is contained in the filter.php file. You will need to set up your Bugzilla email such that you can feed the bugmail into this php script. The way I have it set up is I have my Bugzilla email account redirect to a mailbox on my webhost account (of the form myusername+bugmash@myaccountserver.dreamhost.com), and then I have a .forward file in my webhost account with a line like the one below: + +`"| /home/myusername/path/to/filter.php"` + +This takes all incoming email and pipes it to the filter.php script. Make sure you mark filter.php as executable, and update the hashbang line at the top to point to your php interpreter. The scraper extracts useful information from Bugzilla emails, and stuffs it into a MySQL database (which is the second component, detailed below). If you want to be able to process PGP/GPG-encrypted bugmail as well, you can additionally pipe the mail through decrypt_mail.awk before piping it to filter.php. This assumes you have no passphrase on your gpg key, such that "gpg --decrypt" will successfully decrypt stdin to stdout. In this case, your .forward file might look like this: + +`"| /home/myusername/path/to/decrypt_mail.awk | /home/myusername/path/to/filter.php"` + +The second component is a MySQL database. This is where all the bug data is actually stored after being extracted from the emails. You will need to set up a database with the necessary tables (schemas are in tables.sql). You will also need to rename the example.config.php file to bugmash.config.php and put your database access information into it. While you're at it, also fill in $_ME in the config file with the email address on your Bugzilla account (i.e. what shows up when you go to Bugzilla preferences). If you have multiple Bugzilla accounts you can put them all in. + +**IMPORTANT!** Make sure that the bugmash.config.php file is not in your web server's document root (generally $HOME/www), otherwise your database credentials may be at risk of being snarfed by evil crackers!! + +So, with these two components, bugmail you receive should be getting parsed and inserted correctly. If you are piping emails to the script that are not bugmail, they will be saved to the folder you specify in the config file. If, for whatever reason, the email scraper is unable to parse an actual bugmail, it will save the bugmail, along with a corresponding .err file, to the same directory that filter.php is in. This is so that you don't lose any bugmail, and can improve the scraper to handle the failures. (The most frequent reason this happens is when an unknown bug field changes value.) + +The third component is the front-end, which consists of the common.php, dashboard.php, wipe.php, and search.php files. These should be installed into a directory and served via a normal PHP-supporting webserver (e.g. Apache with mod_php). You will need to modify the $BUGMASH_DIR variable in common.php so that it can find the bugmash.config.php file with the database credentials. Once you have this set up, you can access the dashboard by going to the dashboard.php in your favourite web browser (i.e. Firefox). + +**IMPORTANT!** You should probably lock down access to the dashboard (and the other web-facing .php files) by using .htaccess or some other kind of authentication. Failure to do so could allow evil crackers to walk right in and mark all your bugmail as read!! + +Now, if you have set all of that up, there is one final (optional) component you can install. This consists of the tags.php file, which should be placed in the same web-facing directory as common.php, and the bugtags.user.js user script. This user script can be installed into Scriptish (a Firefox add-on) and modifies Bugzilla bug listing pages (such as the "My Bugs" page, or any search results page) to display and edit your personal bug tags. Before installing the user script into Scriptish, make sure you modify the TAGS_SERVER variable as appropriate so that it points to your web-facing tags.php file. + +## Usage + +As with most power tools, there are many intricacies to using Bugmash. These are documented here in favour of keeping the actual interface minimal and fast. + +### Bug blocks + +The primary mode of interaction with Bugmash is through the dashboard.php front-end. This will show all of the data from your bugmail that you have not marked as "viewed" at the time that the page is loaded. Note that the page does not auto-update or auto-refresh; you have to reload manually when you want to view the latest bug information. This is by design because I hate pages that move content around under my cursor. Here is a screenshot of what the dashboard looks like: + +
![Screenshot of the dashboard](bugmash-dashboard-screenshot.png)
+ +All of the data for a given bug is contained inside a "bug block" (the thing with the blue header and footer). Note that if a bug is marked "secure" (so that the bugmail is PGP/GPG-encrypted), the header and border of the bug block is red instead. The header contains the bug number, the bug title, and two links in the top-right corner. The first link will be either "N" (for Note) or "U" (for Update). Clicking on this link will take you to the bottom of the dashboard, to the bug notes section, and allow you to add or edit the corresponding note and tag information for that bug. The "N" link will be displayed when there is no pre-existing note/tag, and the "U" link will be displayed when there is already a pre-existing note/tag for the bug. You can also hover your mouse over the "U" link to get the note/tag information in a tooltip. + +The second link is an "X" that marks the bug block as viewed. Clicking on the X makes the bug block disappear and shifts the column up so that you can click on the X for the next bug block without moving your mouse. Note that when you mark a bug block as viewed, the block is hidden and an XMLHttpRequest is sent to the server; if that XMLHttpRequest fails, the bug block will be made visible again, and the "X" will be replaced with "[E]" (for Error). If this happens, you can click on the "[E]" to try again (it does the same thing as the X). + +The data inside the bug block should be largely self-explanatory; it is basically a compressed version of the data you should be accustomed to seeing in normal bugmail. Note that URLs and bug numbers should be clickable even though they are not blue-and-underlined. This is to reduce visual noise for when you have a lot of them. The footer on the bug block has a link to take you back to the top of the bug block; this is useful for when the bug block is taller than a screen, or if you're on a mobile device with a small screen. The other noteworthy thing about bug block data is that comments made by TinderboxPushlog Robot will be greyed out. This allows you to quickly scan a bug block for a randomorange bug and see if there were any "real" comments made. Also something to note is that for better layout of bug data, whitespace in comments is often munged and/or thrown away (this also happens because of the way emails are parsed). + +The bug blocks are arranged into four columns by default (on narrow screens the four columns get stacked into one column such that the leftmost column is on top and the rightmost is at the bottom). The four columns are generally arranged in decreasing order of importance from left to right. The leftmost column contains bugs that have a review requested of you, or contain a response to a review that you requested. The second column is for bugs that are assigned to you, or which you filed. The third column is for bugs you are CC'd on, or have voted on. The final column is for pretty much anything else (usually component-watching). If you would like to change this sorting behaviour, the column() function in dashboard.php is the place to look. + +### Bug notes and tags + +Below all the bug blocks is the bug notes section. Each bug note has a bug number, a note field, a tag field, and the bug title (if available). You can add notes/tags either by using the N/U link on the bug block, or the "Add note" button at the bottom of the section. Use the "Save notes" link to save changes (this will reload the whole page, rather than using XMLHttpRequest). Saving an empty note and tag for a bug removes that row from the notes section. + +The primary difference between notes and tags is that, if you have the bugtags.user.js script installed, the tags will show up in Bugzilla next to the title of the bug on any bug listing page, such as shown in the screenshot below. If you do not have the userscript, then the tags field is redundant. Tags that start with a bang (!) will be displayed by the userscript in red, or blue otherwise. You can click on the tag to edit it; bugs with no tags will have a "+" that you can click to add tags. As tag updates from Bugzilla pages are done via XMLHttpRequest, they are shown in yellow while the update request is pending, and might revert to their old value if the update fails. + +
![Screenshot of bugtags on Bugzilla](bugmash-bugtags-screenshot.png)
+ +**WARNING!** Be careful when editing tags from both the Bugzilla userscript interface and the dashboard interface, as you may inadvertently clobber changes you made. In particular this may happen if you make a change from Bugzilla, and then make a change (possibly to a different bug's tag) from the dashboard without reloading it, and then saving the dashboard bug notes. + +### Search + +Below the bug notes section is the search field. This is a very bare-bones search that searches through your saved bug data. Search queries are split into tokens on whitespace, and only bugs that match all of the tokens are shown. The results are limited to bugs modified in the last 15 days, and are sorted by time (most recent first). Try not to search for really common substrings because there is no limiting on the number of hits other than recency, and the search implementation isn't particularly efficient - it may bring your server to its knees. + +### Handling errors + +Errors may occur if a bugmail cannot be parsed by the scraper. When this happens, the email and a corresponding .err file is saved into the folder where filter.php resides. The number of .err files is shown in browser window title bar when you are viewing the dashboard. This allows you to quickly check if there have been any errors. When an error occurs, you'll need to look at the .err file and the email to see why the email could not be parsed, and patch up the filter.php to handle it. Or you could file a bug against the github repo and get me to do it. + +## Known issues + +* The code to strikethrough obsoleted review requests doesn't seem to be taking effect. Not sure where in the code this bug is, haven't really looked at it. +* When concurrently editing tags via both the user script and the dashboard, it is easy to accidentally clobber tag changes. This should be detected and handled better. +* Bugzilla comments which contain "--" on a single line will be prematurely terminated, since that is taken as the end-of-comment marker. +* Review authors whose name/email wraps onto a second line in the email are not handled properly. + +## Areas for improvement + +* Add a quick-reply form to comment on bugs +* Beef up the search feature +* Hook into the Bugzilla API for more awesome +* Prune data from the database periodically +* A way to undo marking bugs as viewed, in case of accidentally marking a bug as viewed +* A way to specify a list of terms to highlight in bug blocks From a51f6ff1e3190e0907b68c45b621552bfd62e859 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 6 May 2015 08:41:12 -0400 Subject: [PATCH 050/141] Don't scroll down/right when wiping a bug block from the bottom. --- dashboard.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard.php b/dashboard.php index 9d7994e..b9afff6 100644 --- a/dashboard.php +++ b/dashboard.php @@ -273,8 +273,8 @@ function column( &$reasons ) { . '' . '
%s
' . '
' - . 'X ' - . 'Back to top' + . 'X ' + . 'Back to top' . '
' . '', ($meta_secure[ $bug ] ? 'secure ' : ''), From 4b95becc02099214c37c2802586a3c7c12e0f36f Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 22 Jul 2015 17:02:10 -0400 Subject: [PATCH 051/141] Update user script to work in greasemonkey again --- bugtags.user.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bugtags.user.js b/bugtags.user.js index c8e4af9..8794f9a 100644 --- a/bugtags.user.js +++ b/bugtags.user.js @@ -7,7 +7,8 @@ // @description Allows you tag bugs; the tags are then shown on Bugzilla pages // @match https://bugzilla.mozilla.org/* // @run-at document-end -// @grant unsafeWindow +// @grant GM_log +// @grant GM_xmlhttpRequest // ==/UserScript== var TAGS_SERVER = 'https://example.com/path/to/tags.php'; // point this to your tags.php From 575dc28d5667d4be5c9c21e200263801b9ac7d88 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 28 Sep 2015 17:29:50 -0400 Subject: [PATCH 052/141] Add regression range as a keyword --- filter.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/filter.php b/filter.php index a520fb7..b896d22 100644 --- a/filter.php +++ b/filter.php @@ -189,6 +189,10 @@ function normalizeFieldList( $fieldString ) { if ($i + 1 < count( $words ) && $words[ $i + 1 ] == 'Story') { $word .= ' ' . $words[ ++$i ]; } + } else if ($word == 'Has' /* Regression Range */) { + if ($i + 2 < count( $words ) && $words[ $i + 1 ] == 'Regression' && $words[ $i + 2 ] == 'Range') { + $word .= ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ]; + } } if ($word == 'Ever Confirmed') { $word = 'Ever confirmed'; From 2597a484ca8f5aca0ebd063c7186b90a9c501e3c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 2 Nov 2015 12:49:53 -0500 Subject: [PATCH 053/141] Add Has STR as a keyword --- filter.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/filter.php b/filter.php index b896d22..02cac25 100644 --- a/filter.php +++ b/filter.php @@ -189,8 +189,10 @@ function normalizeFieldList( $fieldString ) { if ($i + 1 < count( $words ) && $words[ $i + 1 ] == 'Story') { $word .= ' ' . $words[ ++$i ]; } - } else if ($word == 'Has' /* Regression Range */) { - if ($i + 2 < count( $words ) && $words[ $i + 1 ] == 'Regression' && $words[ $i + 2 ] == 'Range') { + } else if ($word == 'Has' /* STR|Regression Range */) { + if ($i + 1 < count( $words ) && $words[ $i + 1 ] == 'STR') { + $word .= ' ' . $words[ ++$i ]; + } else if ($i + 2 < count( $words ) && $words[ $i + 1 ] == 'Regression' && $words[ $i + 2 ] == 'Range') { $word .= ' ' . $words[ ++$i ] . ' ' . $words[ ++$i ]; } } From 38069c53a1f9cc898e0a15d9fe018a533d334ec5 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 19 Feb 2016 13:07:41 -0500 Subject: [PATCH 054/141] Stop assuming attachments are splinter reviews --- dashboard.php | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/dashboard.php b/dashboard.php index b9afff6..93f45d5 100644 --- a/dashboard.php +++ b/dashboard.php @@ -106,7 +106,7 @@ function linkify( $text, $bug ) { $text = preg_replace( '#(https?://\S+)#i', '$1', $text ); $text = preg_replace( '/(bug\s+)(\d+)/ie', 'buglink(\'\\1\', \'\\2\')', $text ); $text = preg_replace( '/(bug-)(\d+)/ie', 'buglink(\'\\1\', \'\\2\')', $text ); - $text = preg_replace( '/(Attachment #?)(\d+)/i', '$1$2', $text ); + $text = preg_replace( '/(Attachment #?)(\d+)/i', '$1$2', $text ); return $text; } @@ -148,16 +148,28 @@ function column( &$reasons ) { while ($row = $result->fetch_assoc()) { $numRows++; $stamp = strtotime( $row['stamp'] ); - $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s: %s%s %s%s
', - $row['id'], - escapeHTML( $row['author'] ), - abbrevFlag( $row['flag'] ), - ($row['granted'] ? '+' : '-'), - $_BASE_URL, - $row['bug'], - $row['attachment'], - escapeHTML( $row['title'] ), - (strlen( $row['comment'] ) > 0 ? ' with comments: ' . escapeHTML( $row['comment'] ) : '') ) . "\n"; + if (strstr( $row['title'], 'MozReview Request:' )) { + $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s: %s%s %s%s
', + $row['id'], + escapeHTML( $row['author'] ), + abbrevFlag( $row['flag'] ), + ($row['granted'] ? '+' : '-'), + $_BASE_URL, + $row['attachment'], + escapeHTML( $row['title'] ), + (strlen( $row['comment'] ) > 0 ? ' with comments: ' . escapeHTML( $row['comment'] ) : '') ) . "\n"; + } else { + $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s: %s%s %s%s
', + $row['id'], + escapeHTML( $row['author'] ), + abbrevFlag( $row['flag'] ), + ($row['granted'] ? '+' : '-'), + $_BASE_URL, + $row['bug'], + $row['attachment'], + escapeHTML( $row['title'] ), + (strlen( $row['comment'] ) > 0 ? ' with comments: ' . escapeHTML( $row['comment'] ) : '') ) . "\n"; + } $reasons[ $row['bug'] ][] = 'review'; $filterComments[ $row['attachment'] ][] = $row['comment']; From f7a1c89e75dac390cb5282d0092a360523b3c704 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 18 Apr 2016 12:38:55 -0400 Subject: [PATCH 055/141] Update filter to catch 'Restrict Comments' --- filter.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/filter.php b/filter.php index 02cac25..d328e99 100644 --- a/filter.php +++ b/filter.php @@ -166,7 +166,8 @@ function normalizeFieldList( $fieldString ) { || $word == 'Crash' /* Signature */ || $word == 'See' /* Also */ || $word == 'Last' /* Resolved */ - || $word == 'QA' /* Whiteboard */) + || $word == 'QA' /* Whiteboard */ + || $word == 'Restrict' /* Comments */) { if ($i + 1 >= count( $words )) { fail( 'Unrecognized field list (4): ' . print_r( $words, true ) ); From 11ea0c271a96ae26af3997b8e1e5433285fceca2 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 9 Jan 2017 13:40:30 -0500 Subject: [PATCH 056/141] Add support for Mozilla theme --- bugtags.user.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bugtags.user.js b/bugtags.user.js index 8794f9a..aaef1aa 100644 --- a/bugtags.user.js +++ b/bugtags.user.js @@ -14,6 +14,10 @@ var TAGS_SERVER = 'https://example.com/path/to/tags.php'; // point this to your tags.php function getUser() { + var login = document.querySelector('td#moz_login .anchor'); + if (login != null) { + return login.textContent.trim(); + } var links = document.links; for (var i = 0; i < links.length; i++) { if (links[i].href.indexOf( "logout" ) > 0) { From 4af0280ac5bbde20bed2b08d9703fd8d565c05ae Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 3 Feb 2017 15:22:53 -0500 Subject: [PATCH 057/141] Drop nag mails --- filter.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/filter.php b/filter.php index d328e99..5a6a22b 100644 --- a/filter.php +++ b/filter.php @@ -400,8 +400,11 @@ function updateMetadata( $date ) { } } -$bug = getField( 'id' ); $type = getField( 'type' ); +if ($type == 'nag') { + success(); +} +$bug = getField( 'id' ); $date = date( 'Y-m-d H:i:s', getField( 'date' ) ); updateMetadata( $date ); From 48ea1b033a25abb8fc87f48f57c5fe87881c3407 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 26 Sep 2017 12:11:52 -0400 Subject: [PATCH 058/141] Move the user script into bugtags/ --- bugtags.user.js => bugtags/bugtags.user.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bugtags.user.js => bugtags/bugtags.user.js (100%) diff --git a/bugtags.user.js b/bugtags/bugtags.user.js similarity index 100% rename from bugtags.user.js rename to bugtags/bugtags.user.js From c047d50fe640f38b214ed97f517f810c3040d87a Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 26 Sep 2017 12:16:19 -0400 Subject: [PATCH 059/141] Turn bugtags into a webextension --- bugtags/bugtags.user.js | 12 ++++++++++++ bugtags/manifest.json | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 bugtags/manifest.json diff --git a/bugtags/bugtags.user.js b/bugtags/bugtags.user.js index aaef1aa..7825795 100644 --- a/bugtags/bugtags.user.js +++ b/bugtags/bugtags.user.js @@ -13,6 +13,18 @@ var TAGS_SERVER = 'https://example.com/path/to/tags.php'; // point this to your tags.php +function GM_log( thing ) { + console.log( thing ); +} + +function GM_xmlhttpRequest( blob ) { + var xhr = new XMLHttpRequest(); + xhr.onload = blob.onload; + xhr.onerror = blob.onerror; + xhr.open( blob.method, blob.url, true ); + xhr.send( blob.data ); +} + function getUser() { var login = document.querySelector('td#moz_login .anchor'); if (login != null) { diff --git a/bugtags/manifest.json b/bugtags/manifest.json new file mode 100644 index 0000000..f17529d --- /dev/null +++ b/bugtags/manifest.json @@ -0,0 +1,24 @@ +{ + "manifest_version": 2, + "name": "BugTags", + "version": "1.0", + + "description": "Displays tags associated with bugs on bugzilla pages.", + + "applications": { + "gecko": { + "id": "bugtags@staktrace.com" + } + }, + + "content_scripts": [ + { + "matches": ["https://bugzilla.mozilla.org/*"], + "js": ["bugtags.user.js"] + } + ], + + "permissions": [ + "https://example.com/*", + ] +} From 92ae62def7029948a3cfe21e5e001bfd62ef5250 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 26 Sep 2017 13:03:56 -0400 Subject: [PATCH 060/141] Fix migration from GM to standalone content script --- bugtags/bugtags.user.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bugtags/bugtags.user.js b/bugtags/bugtags.user.js index 7825795..1d370bc 100644 --- a/bugtags/bugtags.user.js +++ b/bugtags/bugtags.user.js @@ -19,8 +19,12 @@ function GM_log( thing ) { function GM_xmlhttpRequest( blob ) { var xhr = new XMLHttpRequest(); - xhr.onload = blob.onload; - xhr.onerror = blob.onerror; + xhr.onload = function() { + blob.onload( xhr ); + }; + xhr.onerror = function() { + blob.onerror( xhr ); + }; xhr.open( blob.method, blob.url, true ); xhr.send( blob.data ); } @@ -119,7 +123,7 @@ function insertListBugTags( user, bugnumbers ) { }, onerror: function( res ) { GM_log( "Error fetching bug tags!" ); - GM_log( res.statusText ); + GM_log( res.status ); GM_log( res.responseText ); } }; @@ -155,7 +159,7 @@ function insertTreeBugTags( user, bugnumbers ) { }, onerror: function( res ) { GM_log( "Error fetching bug tags!" ); - GM_log( res.statusText ); + GM_log( res.status ); GM_log( res.responseText ); } }); From 042916176a8b397a464725719e04ea9b93938d9b Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 26 Sep 2017 13:04:26 -0400 Subject: [PATCH 061/141] Drop unnecessary application id --- bugtags/manifest.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/bugtags/manifest.json b/bugtags/manifest.json index f17529d..11aff61 100644 --- a/bugtags/manifest.json +++ b/bugtags/manifest.json @@ -5,12 +5,6 @@ "description": "Displays tags associated with bugs on bugzilla pages.", - "applications": { - "gecko": { - "id": "bugtags@staktrace.com" - } - }, - "content_scripts": [ { "matches": ["https://bugzilla.mozilla.org/*"], From 60e821b52e54b21f720bdfe46e0cd24a3eaee884 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 21 Dec 2017 09:45:19 -0500 Subject: [PATCH 062/141] Add user-finding for new bugzilla layout --- bugtags/bugtags.user.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bugtags/bugtags.user.js b/bugtags/bugtags.user.js index 1d370bc..d3fdc6c 100644 --- a/bugtags/bugtags.user.js +++ b/bugtags/bugtags.user.js @@ -34,6 +34,10 @@ function getUser() { if (login != null) { return login.textContent.trim(); } + var newLogin = document.querySelector('.email'); + if (email != null) { + return email.textContent.trim(); + } var links = document.links; for (var i = 0; i < links.length; i++) { if (links[i].href.indexOf( "logout" ) > 0) { From 2decf5bcabefc35c25bb39754d5f4a85020890b1 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sun, 25 Mar 2018 20:02:47 -0400 Subject: [PATCH 063/141] Add amazonses as a valid sender --- filter.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/filter.php b/filter.php index 5a6a22b..b9906eb 100644 --- a/filter.php +++ b/filter.php @@ -22,9 +22,18 @@ } if (isset( $_CHECK_SENDER ) && intval( $_CHECK_SENDER )) { - if ((! isset( $_SERVER['SENDER'] )) || (strpos( $_SERVER['SENDER'], 'bugzilla' ) !== 0)) { + $senderOk = false; + if (isset( $_SERVER['SENDER'] )) { + if ((strpos( $_SERVER['SENDER'], 'bugzilla' ) !== 0) || + (strpos( $_SERVER['SENDER'], 'amazonses' ) !== 0)) + { + $senderOk = true; + } + } + if (! $senderOk) { // doesn't look like a bugmail, probably spam but possible bounce notifications. put it aside file_put_contents( $_UNFILTERED_DIR . '/' . $filename, $mailString ); + //file_put_contents( $_UNFILTERED_DIR . '/' . $filename . '.sender', isset( $_SERVER['SENDER'] ) ? $_SERVER['SENDER'] : '' ); exit( 0 ); } } From b893e06b934e4b7c83d427983534e35c7505d13a Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 26 Mar 2018 12:49:18 -0400 Subject: [PATCH 064/141] amazonses was the wrong thing to check --- filter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.php b/filter.php index b9906eb..3aa9ffa 100644 --- a/filter.php +++ b/filter.php @@ -25,7 +25,7 @@ $senderOk = false; if (isset( $_SERVER['SENDER'] )) { if ((strpos( $_SERVER['SENDER'], 'bugzilla' ) !== 0) || - (strpos( $_SERVER['SENDER'], 'amazonses' ) !== 0)) + (strpos( $mailText, 'Bugzilla' ) !== 0)) { $senderOk = true; } From 1bbc32a0993637123145a7a25b3cb9b17d031b1c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 29 Mar 2018 09:28:18 -0400 Subject: [PATCH 065/141] Fix checks --- filter.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filter.php b/filter.php index 3aa9ffa..eb5914e 100644 --- a/filter.php +++ b/filter.php @@ -24,8 +24,8 @@ if (isset( $_CHECK_SENDER ) && intval( $_CHECK_SENDER )) { $senderOk = false; if (isset( $_SERVER['SENDER'] )) { - if ((strpos( $_SERVER['SENDER'], 'bugzilla' ) !== 0) || - (strpos( $mailText, 'Bugzilla' ) !== 0)) + if ((stripos( $_SERVER['SENDER'], 'bugzilla' ) !== FALSE) || + (stripos( $mailText, 'bugzilla' ) !== FALSE)) { $senderOk = true; } From 272cade81c26094b4474c6850b522a298273d09b Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 24 May 2018 13:21:05 -0400 Subject: [PATCH 066/141] Fix stupid PHP warnings about undefined indices --- dashboard.php | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/dashboard.php b/dashboard.php index 93f45d5..c64b4a6 100644 --- a/dashboard.php +++ b/dashboard.php @@ -138,6 +138,19 @@ function column( &$reasons ) { } } +function initEmpty( &$blocks, $bug, $stamp ) { + if (!isset( $blocks[ $bug ][ $stamp ])) { + $blocks[ $bug ][ $stamp ] = ''; + } +} + +function safeGet( &$array, $index ) { + if (isset( $array[ $index ] )) { + return $array[ $index ]; + } + return ''; +} + $filterComments = array(); $filterFlags = array(); $numRows = 0; @@ -148,6 +161,7 @@ function column( &$reasons ) { while ($row = $result->fetch_assoc()) { $numRows++; $stamp = strtotime( $row['stamp'] ); + initEmpty( $bblocks, $row['bug'], $stamp ); if (strstr( $row['title'], 'MozReview Request:' )) { $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s: %s%s %s%s
', $row['id'], @@ -180,6 +194,7 @@ function column( &$reasons ) { while ($row = $result->fetch_assoc()) { $numRows++; $stamp = strtotime( $row['stamp'] ); + initEmpty( $bblocks, $row['bug'], $stamp ); $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s%s? %s%s
', $row['id'], ($row['cancelled'] ? '' : ''), @@ -200,6 +215,7 @@ function column( &$reasons ) { while ($row = $result->fetch_assoc()) { $numRows++; $stamp = strtotime( $row['stamp'] ); + initEmpty( $bblocks, $row['bug'], $stamp ); $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
New: %s by %s
%s
', $row['id'], $_BASE_URL, @@ -230,6 +246,7 @@ function column( &$reasons ) { $numRows++; $stamp = strtotime( $row['stamp'] ); + initEmpty( $bblocks, $row['bug'], $stamp ); $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s: %s → %s
', ($hide ? ' style="display: none"' : ''), $row['id'], @@ -263,6 +280,7 @@ function column( &$reasons ) { $stamp = strtotime( $row['stamp'] ); $isTbplRobot = (stripWhitespace( $row['author'] ) == 'TBPLRobot') || (stripWhitespace( $row['author'] ) == 'TreeherderRobot'); + initEmpty( $bblocks, $row['bug'], $stamp ); $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s said:
%s
', ($hide ? 'display: none;' : 'white-space: pre-line;'), ($isTbplRobot ? 'opacity: 0.5;' : ''), @@ -527,10 +545,10 @@ function noteify( linkElement, bugnumber ) { $bug, $bug, $bug, - escapeHTML( $meta_notes[ $bug ] ), + escapeHTML( safeGet( $meta_notes, $bug ) ), $bug, - escapeHTML( $meta_tags[ $bug ] ), - escapeHTML( $meta_titles[ $bug ] ) ), + escapeHTML( safeGet( $meta_tags, $bug ) ), + escapeHTML( safeGet( $meta_titles, $bug ) ) ), "\n"; } ?> From 37bbc3045eb5f46fa3183b78353f3c5596c6adb4 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 24 May 2018 13:26:30 -0400 Subject: [PATCH 067/141] More PHP warning fixes --- dashboard.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dashboard.php b/dashboard.php index c64b4a6..15e10a0 100644 --- a/dashboard.php +++ b/dashboard.php @@ -307,7 +307,7 @@ function safeGet( &$array, $index ) { . 'Back to top' . '' . '', - ($meta_secure[ $bug ] ? 'secure ' : ''), + (empty( $meta_secure[ $bug ] ) ? '' : 'secure '), $bug, (in_array($bug, $bugsWithNotes) ? escapeHTML( $meta_notes[ $bug ] . ' | ' . $meta_tags[ $bug ] ) : ''), $bug, @@ -321,7 +321,9 @@ function safeGet( &$array, $index ) { $bug, $bug, $bug ) . "\n"; - $columns[ column( $reasons[ $bug ] ) ][ $touchTime ] .= $block; + $col = column( $reasons[ $bug ] ); + initEmpty( $columns, $col, $touchTime ); + $columns[ $col ][ $touchTime ] .= $block; } $_DB->close(); From 8d30ec98c31a1d5856f2114d77014bb39323e73c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 24 May 2018 13:28:04 -0400 Subject: [PATCH 068/141] More warning fix --- dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard.php b/dashboard.php index 15e10a0..d5dd2c3 100644 --- a/dashboard.php +++ b/dashboard.php @@ -522,7 +522,7 @@ function noteify( linkElement, bugnumber ) { ', "\n"; if (count( $buglist ) > 0) { ksort( $buglist, SORT_NUMERIC ); From 384e052e0b9cb20342d224e367bd5bc706a17419 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 24 May 2018 13:30:04 -0400 Subject: [PATCH 069/141] Fix typo --- bugtags/bugtags.user.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bugtags/bugtags.user.js b/bugtags/bugtags.user.js index d3fdc6c..f1e0931 100644 --- a/bugtags/bugtags.user.js +++ b/bugtags/bugtags.user.js @@ -35,8 +35,8 @@ function getUser() { return login.textContent.trim(); } var newLogin = document.querySelector('.email'); - if (email != null) { - return email.textContent.trim(); + if (newLogin != null) { + return newLogin.textContent.trim(); } var links = document.links; for (var i = 0; i < links.length; i++) { From 2760067aa200d1658aa2671f1a9b39fe925e6395 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 6 Jun 2018 17:48:49 -0400 Subject: [PATCH 070/141] Remove unused variable --- search.php | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/search.php b/search.php index c8eb9e4..c2f0ddc 100644 --- a/search.php +++ b/search.php @@ -184,12 +184,10 @@ function formatHits( $text, $terms, $isTitle ) { } else { $timestamps[ $matchRow['bug'] ] = max( $timestamps[ $matchRow['bug'] ], $timestamp ); } - if (! $metaHit) { - foreach ($_SEARCH_COLUMNS[ $matchRow['table'] ] AS $column) { - $hit = formatHits( $matchRow[ $column ], $terms, false ); - if ($hit) { - $results[ $matchRow['bug'] ][] = $hit; - } + foreach ($_SEARCH_COLUMNS[ $matchRow['table'] ] AS $column) { + $hit = formatHits( $matchRow[ $column ], $terms, false ); + if ($hit) { + $results[ $matchRow['bug'] ][] = $hit; } } } From dec066ed0e0cb4ad3c296b8ca580c05502809df0 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 23 Jul 2018 22:43:37 -0400 Subject: [PATCH 071/141] Reorganize files into folders --- tables.sql => schemas/tables.sql | 0 example.config.php => scraper/config.php.sample | 0 decrypt_mail.awk => scraper/decrypt_mail.awk | 0 filter.php => scraper/filter.php | 2 +- common.php => www/common.php | 4 ++-- dashboard.php => www/dashboard.php | 0 search.php => www/search.php | 0 tags.php => www/tags.php | 0 wipe.php => www/wipe.php | 0 9 files changed, 3 insertions(+), 3 deletions(-) rename tables.sql => schemas/tables.sql (100%) rename example.config.php => scraper/config.php.sample (100%) rename decrypt_mail.awk => scraper/decrypt_mail.awk (100%) rename filter.php => scraper/filter.php (99%) rename common.php => www/common.php (90%) rename dashboard.php => www/dashboard.php (100%) rename search.php => www/search.php (100%) rename tags.php => www/tags.php (100%) rename wipe.php => www/wipe.php (100%) diff --git a/tables.sql b/schemas/tables.sql similarity index 100% rename from tables.sql rename to schemas/tables.sql diff --git a/example.config.php b/scraper/config.php.sample similarity index 100% rename from example.config.php rename to scraper/config.php.sample diff --git a/decrypt_mail.awk b/scraper/decrypt_mail.awk similarity index 100% rename from decrypt_mail.awk rename to scraper/decrypt_mail.awk diff --git a/filter.php b/scraper/filter.php similarity index 99% rename from filter.php rename to scraper/filter.php index eb5914e..c7de8c1 100644 --- a/filter.php +++ b/scraper/filter.php @@ -1,7 +1,7 @@ #!/usr/bin/env php Date: Mon, 23 Jul 2018 22:44:12 -0400 Subject: [PATCH 072/141] Add gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4dbb059 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +scraper/config.php From cf0df7ad7c94e3671cb1941339d1c37c4b40bc32 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 23 Jul 2018 22:48:59 -0400 Subject: [PATCH 073/141] Add Due Date field --- scraper/filter.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scraper/filter.php b/scraper/filter.php index c7de8c1..5bfbb5f 100644 --- a/scraper/filter.php +++ b/scraper/filter.php @@ -176,7 +176,8 @@ function normalizeFieldList( $fieldString ) { || $word == 'See' /* Also */ || $word == 'Last' /* Resolved */ || $word == 'QA' /* Whiteboard */ - || $word == 'Restrict' /* Comments */) + || $word == 'Restrict' /* Comments */ + || $word == 'Due' /* Date */) { if ($i + 1 >= count( $words )) { fail( 'Unrecognized field list (4): ' . print_r( $words, true ) ); From 1e481453b77be9c9b68e189bf82544471e911f3f Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 25 Jul 2018 22:26:09 -0400 Subject: [PATCH 074/141] Add nginx config --- nginx/nginx.conf | 31 +++++++++++++++++++++++++++++++ www/index.html | 1 + 2 files changed, 32 insertions(+) create mode 100644 nginx/nginx.conf create mode 100644 www/index.html diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..e1d4dd7 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,31 @@ +server { + listen [::]:80; + listen 80; + server_name bugmash.staktrace.com; + return 301 https://$host$request_uri; +} + +server { + listen [::]:443; + listen 443; + server_name bugmash.staktrace.com; + root /ebs/bugmash/www; + index index.php index.html index.htm; + + include includes/ssl.inc; + ssl_certificate /usr/local/etc/letsencrypt/live/bugmash.staktrace.com/fullchain.pem; + ssl_certificate_key /usr/local/etc/letsencrypt/live/bugmash.staktrace.com/privkey.pem; + + location / { + try_files $uri $uri/ =404; + } + + location ~ \.php$ { + auth_basic "Bugmash"; + auth_basic_user_file "/ebs/bugmash/www-conf/bugmash.htpass"; + try_files $uri =404; + include includes/php-inner.inc; + } + + include includes/error.inc; +} diff --git a/www/index.html b/www/index.html new file mode 100644 index 0000000..2be5171 --- /dev/null +++ b/www/index.html @@ -0,0 +1 @@ +Bugmash! From 1d9c2a4747f65f56e64e5a319ca1cda12d9705cb Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 25 Jul 2018 22:39:45 -0400 Subject: [PATCH 075/141] Fix filename --- scraper/config.php.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scraper/config.php.sample b/scraper/config.php.sample index ae2160d..eb4d423 100644 --- a/scraper/config.php.sample +++ b/scraper/config.php.sample @@ -1,6 +1,6 @@ Date: Sat, 28 Jul 2018 22:04:28 -0400 Subject: [PATCH 076/141] Add more scraper tools --- scraper/cron-pump.sh | 28 ++++++++++++++++++++++++++++ scraper/filter.php | 0 scraper/postfix.forward.symtarget | 1 + scraper/pump.sh | 4 ++++ 4 files changed, 33 insertions(+) create mode 100644 scraper/cron-pump.sh mode change 100644 => 100755 scraper/filter.php create mode 100644 scraper/postfix.forward.symtarget create mode 100755 scraper/pump.sh diff --git a/scraper/cron-pump.sh b/scraper/cron-pump.sh new file mode 100644 index 0000000..943104d --- /dev/null +++ b/scraper/cron-pump.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +pushd $HOME/scraper >/dev/null 2>&1 +for i in $(grep -l "Prepared statement needs to be re-prepared" *.err 2>/dev/null); do + ./pump.sh ${i%.err} + rm $i ${i%.err} +done +for i in $(grep -l "Can't connect to MySQL server on 'db.staktrace.com'" *.err 2>/dev/null); do + ./pump.sh ${i%.err} + rm $i ${i%.err} +done +for i in $(grep -l "Lost connection to MySQL server at 'reading authorization packet'" *.err 2>/dev/null); do + ./pump.sh ${i%.err} + rm $i ${i%.err} +done +for i in $(grep -l "MySQL server has gone away" *.err 2>/dev/null); do + ./pump.sh ${i%.err} + rm $i ${i%.err} +done +for i in $(grep -l "Error connecting to db:" *.err 2>/dev/null); do + ./pump.sh ${i%.err} + rm $i ${i%.err} +done +for i in $(grep -l "into DB" *.err 2>/dev/null); do + ./pump.sh ${i%.err} + rm $i ${i%.err} +done +popd >/dev/null 2>&1 diff --git a/scraper/filter.php b/scraper/filter.php old mode 100644 new mode 100755 diff --git a/scraper/postfix.forward.symtarget b/scraper/postfix.forward.symtarget new file mode 100644 index 0000000..dac2229 --- /dev/null +++ b/scraper/postfix.forward.symtarget @@ -0,0 +1 @@ +"| $HOME/scraper/decrypt_mail.awk | $HOME/scraper/filter.php" diff --git a/scraper/pump.sh b/scraper/pump.sh new file mode 100755 index 0000000..9b0c081 --- /dev/null +++ b/scraper/pump.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +export SENDER=bugzilla-daemon@mozilla.org +cat $1 | ./decrypt_mail.awk | ./filter.php From 6bf2bfd4604da319aa07ffd453bd1a3e699c642d Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 2 Aug 2018 09:54:42 -0400 Subject: [PATCH 077/141] Fail on metadata errors --- scraper/filter.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scraper/filter.php b/scraper/filter.php index 5bfbb5f..66ff246 100755 --- a/scraper/filter.php +++ b/scraper/filter.php @@ -405,8 +405,12 @@ function updateMetadata( $date ) { if (preg_match( '/\[Bug (\d+)\] (.*)( : \[Attachment.*)?$/sU', getField( 'subject' ), $matches ) > 0) { $stmt = prepare( 'INSERT INTO metadata (bug, stamp, title, secure) VALUES (?, ?, ?, ?) ' . 'ON DUPLICATE KEY UPDATE stamp=VALUES(stamp), title=VALUES(title), secure=VALUES(secure)' ); - $stmt->bind_param( 'issi', $matches[1], $date, $matches[2], $bugIsSecure ); - $stmt->execute(); + if (!$stmt->bind_param( 'issi', $matches[1], $date, $matches[2], $bugIsSecure )) { + fail( "Binding params failed for metadata: [{$stmt->error}]" ); + } + if (!$stmt->execute()) { + fail( "Executing statement failed for metadata: [{$stmt->error}]" ); + } } } From c978c932d79e30ae99da00e4cd41f2fcb38401e6 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 2 Aug 2018 09:55:00 -0400 Subject: [PATCH 078/141] Provide a note field --- scraper/filter.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scraper/filter.php b/scraper/filter.php index 66ff246..655b385 100755 --- a/scraper/filter.php +++ b/scraper/filter.php @@ -403,9 +403,10 @@ function updateMetadata( $date ) { global $bugIsSecure; $matches = array(); if (preg_match( '/\[Bug (\d+)\] (.*)( : \[Attachment.*)?$/sU', getField( 'subject' ), $matches ) > 0) { - $stmt = prepare( 'INSERT INTO metadata (bug, stamp, title, secure) VALUES (?, ?, ?, ?) ' + $stmt = prepare( 'INSERT INTO metadata (bug, stamp, title, secure, note) VALUES (?, ?, ?, ?, ?) ' . 'ON DUPLICATE KEY UPDATE stamp=VALUES(stamp), title=VALUES(title), secure=VALUES(secure)' ); - if (!$stmt->bind_param( 'issi', $matches[1], $date, $matches[2], $bugIsSecure )) { + $note = ""; + if (!$stmt->bind_param( 'issis', $matches[1], $date, $matches[2], $bugIsSecure, $note )) { fail( "Binding params failed for metadata: [{$stmt->error}]" ); } if (!$stmt->execute()) { From 7c3b834cb11c17619fc45a4a529ed0fd76726b37 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 2 Aug 2018 10:38:47 -0400 Subject: [PATCH 079/141] Reduce PHP warnings --- bugtags/bugtags.user.js | 2 +- bugtags/manifest.json | 4 ++-- www/dashboard.php | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bugtags/bugtags.user.js b/bugtags/bugtags.user.js index f1e0931..8555a30 100644 --- a/bugtags/bugtags.user.js +++ b/bugtags/bugtags.user.js @@ -11,7 +11,7 @@ // @grant GM_xmlhttpRequest // ==/UserScript== -var TAGS_SERVER = 'https://example.com/path/to/tags.php'; // point this to your tags.php +var TAGS_SERVER = 'https://bugmash.staktrace.com/tags.php'; // point this to your tags.php function GM_log( thing ) { console.log( thing ); diff --git a/bugtags/manifest.json b/bugtags/manifest.json index 11aff61..3c62015 100644 --- a/bugtags/manifest.json +++ b/bugtags/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "BugTags", - "version": "1.0", + "version": "1.2", "description": "Displays tags associated with bugs on bugzilla pages.", @@ -13,6 +13,6 @@ ], "permissions": [ - "https://example.com/*", + "https://bugmash.staktrace.com/*" ] } diff --git a/www/dashboard.php b/www/dashboard.php index d5dd2c3..af77585 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -2,6 +2,8 @@ include_once( 'common.php' ); +date_default_timezone_set( 'UTC' ); + $_DB = new mysqli( $_MYSQL_HOST, $_MYSQL_USER, $_MYSQL_PASS, $_MYSQL_DB ); if (mysqli_connect_errno()) { fail( 'Error connecting to db: ' . mysqli_connect_error() ); @@ -315,7 +317,7 @@ function safeGet( &$array, $index ) { $_BASE_URL, $bug, $bug, - escapeHTML( $meta_titles[ $bug ] ), + escapeHTML( safeGet( $meta_titles, $bug ) ), implode( "\n", $block ), $bug, $bug, From 7606751109ba787db5edcea7943aa691bf257054 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sat, 1 Sep 2018 09:19:49 -0400 Subject: [PATCH 080/141] Avoid PHP warning --- www/dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index af77585..028ef20 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -100,7 +100,7 @@ function stripWhitespace( $stuff ) { function buglink( $prefix, $bug ) { global $_BASE_URL, $meta_titles; - return '' . $prefix . $bug . ''; + return '' . $prefix . $bug . ''; } function linkify( $text, $bug ) { From 83cf0a25a3f17b022935cc52a96c606f47dc945f Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 13 Sep 2018 17:37:25 -0400 Subject: [PATCH 081/141] Add a table for GH issue comments --- schemas/tables.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/schemas/tables.sql b/schemas/tables.sql index 9bff92e..4679d10 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -76,6 +76,22 @@ CREATE TABLE `newbugs` ( KEY (`viewed`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; +CREATE TABLE `gh_issues` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `repo` varchar(255) NOT NULL, + `issue` int(10) unsigned NOT NULL, + `stamp` datetime NOT NULL, + `viewed` tinyint(1) NOT NULL DEFAULT 0, + `reason` varchar(10) NOT NULL, + `commentnum` int(10) NOT NULL, + `author` varchar(255) NOT NULL, + `comment` mediumtext NOT NULL, + PRIMARY KEY (`id`), + KEY (`repo`, `issue`), + KEY (`stamp`), + KEY (`viewed`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + CREATE TABLE `metadata` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `bug` int(10) unsigned NOT NULL DEFAULT 0, From adc94f30ebf48327bae8d28e7a70714eef0e9edf Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 13 Sep 2018 21:59:51 -0400 Subject: [PATCH 082/141] Initial GH email scraper --- .gitignore | 1 + scraper/Cargo.lock | 675 ++++++++++++++++++++++++++++++++++++++++++++ scraper/Cargo.toml | 8 + scraper/src/main.rs | 134 +++++++++ 4 files changed, 818 insertions(+) create mode 100644 scraper/Cargo.lock create mode 100644 scraper/Cargo.toml create mode 100644 scraper/src/main.rs diff --git a/.gitignore b/.gitignore index 4dbb059..3ab10db 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ scraper/config.php +scraper/target diff --git a/scraper/Cargo.lock b/scraper/Cargo.lock new file mode 100644 index 0000000..8d700ae --- /dev/null +++ b/scraper/Cargo.lock @@ -0,0 +1,675 @@ +[[package]] +name = "aho-corasick" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "arrayref" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "atoi" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "base64" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bit-vec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "block-buffer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bufstream" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byte-tools" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byteorder" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cc" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "chrono" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "digest" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding-index-japanese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding-index-korean 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding-index-simpchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding-index-singlebyte 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding-index-japanese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding-index-korean" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding-index-simpchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding-index-singlebyte" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding-index-tradchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding_index_tests" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "flate2" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fnv" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "generic-array" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "idna" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itoa" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lazy_static" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "version_check 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.2.43" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mailparse" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "quoted_printable 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "miniz-sys" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mysql" +version = "14.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bit-vec 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bufstream 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "mysql_common 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "named_pipe 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.78 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "twox-hash 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mysql_common" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atoi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", + "bit-vec 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.78 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", + "sha1 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "twox-hash 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "named_pipe" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "net2" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nix" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-bigint" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-integer" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "quoted_printable" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "redox_syscall" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "regex" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ryu" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "safemem" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "scraper" +version = "0.1.0" +dependencies = [ + "mailparse 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "mysql 14.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde_json" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.78 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "sha2" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)", + "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "smallvec" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thread_local" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "twox-hash" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "typenum" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ucd-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "url" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "utf8-ranges" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "uuid" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "version_check" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a" +"checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" +"checksum atoi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "baa5dc0129ce09c8c87e2714a0b67c095d4a5e3261a7bbb3d7ac44d43d5dd190" +"checksum base64 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "85415d2594767338a74a30c1d370b2f3262ec1b4ed2d7bba5b3faf4de40467d9" +"checksum bit-vec 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4440d5cb623bb7390ae27fec0bb6c61111969860f8e3ae198bfa0663645e67cf" +"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" +"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" +"checksum bufstream 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f2f382711e76b9de6c744cc00d0497baba02fb00a787f088c879f01d09468e32" +"checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" +"checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" +"checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" +"checksum cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3" +"checksum chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "45912881121cb26fad7c38c17ba7daa18764771836b34fab7d3fbd93ed633878" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +"checksum digest 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)" = "5b29c278aa8fd30796bd977169e8004b4aa88cdcd2f32a6eb22bc2d5d38df94a" +"checksum encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" +"checksum encoding-index-japanese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" +"checksum encoding-index-korean 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81" +"checksum encoding-index-simpchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7" +"checksum encoding-index-singlebyte 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" +"checksum encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" +"checksum encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" +"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +"checksum flate2 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "37847f133aae7acf82bb9577ccd8bda241df836787642654286e79679826a54b" +"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" +"checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +"checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" +"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +"checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7" +"checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" +"checksum mailparse 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d1d5ea71343df3c508b245824ec17f5c6f86f79f99ce999bf6ac51edf0465685" +"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +"checksum memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a3b4142ab8738a78c51896f704f83c11df047ff1bda9a92a661aa6361552d93d" +"checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" +"checksum mysql 14.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e5637e744f6b4512cb89e203b04af2bd2157807561bc00a6de29ff5dc9df5d25" +"checksum mysql_common 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4b24fd8e687c238438e3a62cfd5c22558be340077eb1597f02c20be29bad617e" +"checksum named_pipe 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ed10a5ac4f5f7e5d75552b12c1d5d542debca81e573279dd1e4c19fde6efa6d" +"checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" +"checksum nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d37e713a259ff641624b6cb20e3b12b2952313ba36b6823c0f16e6cfd9e5de17" +"checksum num-bigint 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3eceac7784c5dc97c2d6edf30259b4e153e6e2b42b3c85e9a6e9f45d06caef6e" +"checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" +"checksum num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "630de1ef5cc79d0cdd78b7e33b81f083cbfe90de0f4b2b2f07f905867c70e9fe" +"checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +"checksum quoted_printable 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4126fa98c6d7b166e6a29a24ab96721d618759d803df6a8cb35d6140da475b5a" +"checksum rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e464cd887e869cddcae8792a4ee31d23c7edd516700695608f5b98c67ee0131c" +"checksum rand_core 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "edecf0f94da5551fc9b492093e30b041a891657db7940ee221f9d2f66e82eef2" +"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" +"checksum regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2069749032ea3ec200ca51e4a31df41759190a88edca0d2d86ee8bedf7073341" +"checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d" +"checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7" +"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" +"checksum serde 1.0.78 (registry+https://github.com/rust-lang/crates.io-index)" = "92ec94e2754699adddbbc4f555791bd3acc2a2f5574cba16c93a4a9cf4a04415" +"checksum serde_json 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)" = "59790990c5115d16027f00913e2e66de23a51f70422e549d2ad68c8c5f268f1c" +"checksum sha1 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +"checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" +"checksum smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "153ffa32fd170e9944f7e0838edf824a754ec4c1fc64746fcc9fe1f8fa602e5d" +"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +"checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" +"checksum twox-hash 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4f85be565a110ed72ed7048cf56570db04ce0a592c98aa59b7dacde3e5718750" +"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" +"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" +"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +"checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25" +"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +"checksum url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2a321979c09843d272956e73700d12c4e7d3d92b2ee112b31548aef0d4efc5a6" +"checksum utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd70f467df6810094968e2fce0ee1bd0e87157aceb026a8c083bcf5e25b9efe4" +"checksum uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e1436e58182935dcd9ce0add9ea0b558e8a87befe01c1a301e6020aeb0876363" +"checksum version_check 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7716c242968ee87e5542f8021178248f267f295a5c4803beae8b8b7fd9bc6051" +"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" +"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/scraper/Cargo.toml b/scraper/Cargo.toml new file mode 100644 index 0000000..7dd5c87 --- /dev/null +++ b/scraper/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "scraper" +version = "0.1.0" +authors = ["Kartikaya Gupta "] + +[dependencies] +mailparse = "0.6.2" +mysql = "14.1.0" diff --git a/scraper/src/main.rs b/scraper/src/main.rs new file mode 100644 index 0000000..e36cc30 --- /dev/null +++ b/scraper/src/main.rs @@ -0,0 +1,134 @@ +extern crate mailparse; +#[macro_use] +extern crate mysql; + +use std::collections::hash_map::DefaultHasher; +use std::env; +use std::fs::File; +use std::hash::Hasher; +use std::io::{Read, stdin, Write}; +use std::process; +use std::time::SystemTime; + +use mailparse::{dateparse, MailHeaderMap, MailParseError, ParsedMail}; + +fn fail(data: &[u8], msg: &str, err: String) -> ! { + let time = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); + let mut hasher = DefaultHasher::new(); + hasher.write(data); + let uniq_name = format!("{}.{:x}", time, hasher.finish()); + { + let mut file = File::create(&uniq_name).unwrap(); + file.write(data).ok(); + } + let err_name = format!("{}.err", uniq_name); + { + let mut file = File::create(err_name).unwrap(); + writeln!(file, "{}", msg).ok(); + writeln!(file, "{:?}", err).ok(); + } + process::exit(0); +} + +fn get_db() -> Result { + let mut mycnf = match env::home_dir() { + Some(path) => path, + None => return Err("Home dir not found".to_string()), + }; + mycnf.push(".bugmash"); + mycnf.push("scraper.mysql.cnf"); + let mut file = File::open(mycnf).map_err(|e| format!("{:?}", e))?; + let mut connstr = String::new(); + let _ = file.read_to_string(&mut connstr).map_err(|e| format!("{:?}", e))?; + mysql::Pool::new(connstr).map_err(|e| format!("{:?}", e)) +} + +fn get_plain_body(mail: &ParsedMail) -> Result, MailParseError> { + if mail.ctype.mimetype == "text/plain" { + return mail.get_body().map(Some); + } + for subpart in &mail.subparts { + if let Some(x) = get_plain_body(subpart)? { + return Ok(Some(x)); + } + } + Ok(None) +} + +fn url_parts(footer: String) -> Option<(String, String, String)> { + let prefix = "https://github.com/"; + let issues = "/issues/"; + let comment = "#issuecomment-"; + + let repo_ix = footer.find(prefix)? + prefix.len(); + let issues_ix = footer[repo_ix..].find(issues)? + repo_ix; + let comment_ix = footer[issues_ix..].find(comment)? + issues_ix; + let end_ix = footer[comment_ix..].find("\r\n")? + comment_ix; + + return Some((String::from(&footer[repo_ix..issues_ix]), + String::from(&footer[issues_ix + issues.len()..comment_ix]), + String::from(&footer[comment_ix + comment.len()..end_ix]))); +} + +fn split_footer(msg: String) -> (String, Option) { + match msg.find("\r\n-- \r\nYou are receiving this because") { + Some(ix) => (String::from(&msg[0..ix]), Some(String::from(&msg[ix..]))), + None => (msg, None), + } +} + +fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { + let sender = mail.headers.get_first_value("X-GitHub-Sender").map_err(|e| format!("{:?}", e))?.unwrap_or("Unknown".to_string()); + let reason = match mail.headers.get_first_value("X-GitHub-Reason").map_err(|e| format!("{:?}", e))? { + Some(ref s) if s == "mention" => "CC", + _ => "Watch", + }; + let stamp = mail.headers.get_first_value("Date").map_err(|e| format!("{:?}", e))?.map(|v| dateparse(&v).unwrap_or(0)).unwrap_or(0); + let (comment, footer) = match get_plain_body(mail).map_err(|e| format!("{:?}", e))? { + Some(x) => split_footer(x), + None => return Err("No plaintext body found".to_string()), + }; + let footer = footer.ok_or("Unable to find footer".to_string())?; + let (repo, issue, commentnum) = url_parts(footer).ok_or("Unable to extract URL parts".to_string())?; + + println!("Sender: {}", sender); + println!("Reason: {}", reason); + println!("Stamp: {}", stamp); + println!("Comment: {}", comment); + println!("Repo: {}", repo); + println!("Issue: {}", issue); + println!("CommentNum: {}", commentnum); + + let db = get_db()?; + let result = db.prep_exec(r"INSERT INTO gh_issues (repo, issue, stamp, reason, commentnum, author, comment) + VALUES (:repo, :issue, :stamp, :reason, :commentnum, :sender, :comment)", params! { + repo, + issue, + stamp, + reason, + commentnum, + sender, + comment, + }).map_err(|e| format!("{:?}", e))?; + if result.affected_rows() != 1 { + return Err(format!("Affected row count was {}, not 1", result.affected_rows())); + } + Ok(()) +} + +fn main() { + let mut input = Vec::new(); + { + let stdin = stdin(); + let mut handle = stdin.lock(); + let len = handle.read_to_end(&mut input); + len.err().map(|e| fail(&input, "Reading stdin failed", format!("{:?}", e))); + } + + let mail = mailparse::parse_mail(&input).unwrap_or_else(|e| fail(&input, "Unable to parse mail", format!("{:?}", e))); + + let github_reason = mail.headers.get_first_value("X-GitHub-Reason").unwrap_or_else(|e| fail(&input, "Unable to read mail header", format!("{:?}", e))); + if github_reason.is_some() { + return scrape_github_mail(&mail).unwrap_or_else(|e| fail(&input, "Error while scraping mail", e)); + } +} From 3c0e3b4a2b2386c115ba2a79271b9d2c6afb6c26 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 14 Sep 2018 02:12:57 +0000 Subject: [PATCH 083/141] Add sample for scraper mysql config --- .gitignore | 1 + scraper/scraper.mysql.cnf.sample | 1 + 2 files changed, 2 insertions(+) create mode 100644 scraper/scraper.mysql.cnf.sample diff --git a/.gitignore b/.gitignore index 3ab10db..c85e87f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ scraper/config.php +scraper/scraper.mysql.cnf scraper/target diff --git a/scraper/scraper.mysql.cnf.sample b/scraper/scraper.mysql.cnf.sample new file mode 100644 index 0000000..b849ca9 --- /dev/null +++ b/scraper/scraper.mysql.cnf.sample @@ -0,0 +1 @@ +mysql://root:password@localhost:3307/mysql From c392e942f09b7034e672d9044175535b5ef564ae Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 14 Sep 2018 02:38:43 +0000 Subject: [PATCH 084/141] Fix for rust 1.27 compat --- scraper/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index e36cc30..0228ba0 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -8,12 +8,12 @@ use std::fs::File; use std::hash::Hasher; use std::io::{Read, stdin, Write}; use std::process; -use std::time::SystemTime; +use std::time::{SystemTime, UNIX_EPOCH}; use mailparse::{dateparse, MailHeaderMap, MailParseError, ParsedMail}; fn fail(data: &[u8], msg: &str, err: String) -> ! { - let time = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); + let time = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); let mut hasher = DefaultHasher::new(); hasher.write(data); let uniq_name = format!("{}.{:x}", time, hasher.finish()); From 3b72446a17b99b63e1f41290cf6d8a0093658e62 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 14 Sep 2018 02:38:59 +0000 Subject: [PATCH 085/141] Remove test output and fix insertion --- scraper/src/main.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 0228ba0..e6a2ef6 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -91,17 +91,9 @@ fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { let footer = footer.ok_or("Unable to find footer".to_string())?; let (repo, issue, commentnum) = url_parts(footer).ok_or("Unable to extract URL parts".to_string())?; - println!("Sender: {}", sender); - println!("Reason: {}", reason); - println!("Stamp: {}", stamp); - println!("Comment: {}", comment); - println!("Repo: {}", repo); - println!("Issue: {}", issue); - println!("CommentNum: {}", commentnum); - let db = get_db()?; let result = db.prep_exec(r"INSERT INTO gh_issues (repo, issue, stamp, reason, commentnum, author, comment) - VALUES (:repo, :issue, :stamp, :reason, :commentnum, :sender, :comment)", params! { + VALUES (:repo, :issue, FROM_UNIXTIME(:stamp), :reason, :commentnum, :sender, :comment)", params! { repo, issue, stamp, From 6abcc508a1a0da753cb53d036b482ef032c60678 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sun, 16 Sep 2018 22:52:16 -0400 Subject: [PATCH 086/141] Add review_requested category --- scraper/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index e6a2ef6..5b47a56 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -80,6 +80,7 @@ fn split_footer(msg: String) -> (String, Option) { fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { let sender = mail.headers.get_first_value("X-GitHub-Sender").map_err(|e| format!("{:?}", e))?.unwrap_or("Unknown".to_string()); let reason = match mail.headers.get_first_value("X-GitHub-Reason").map_err(|e| format!("{:?}", e))? { + Some(ref s) if s == "review_requested" => "review", Some(ref s) if s == "mention" => "CC", _ => "Watch", }; From ca35352920ed242c6ffa42c4d1eec4107f22d338 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Wed, 26 Sep 2018 14:48:12 +0000 Subject: [PATCH 087/141] Fix more PHP warnings --- www/dashboard.php | 2 +- www/search.php | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index 028ef20..c7cf220 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -311,7 +311,7 @@ function safeGet( &$array, $index ) { . '', (empty( $meta_secure[ $bug ] ) ? '' : 'secure '), $bug, - (in_array($bug, $bugsWithNotes) ? escapeHTML( $meta_notes[ $bug ] . ' | ' . $meta_tags[ $bug ] ) : ''), + (in_array($bug, $bugsWithNotes) ? escapeHTML( safeGet( $meta_notes, $bug ) . ' | ' . safeGet( $meta_tags, $bug ) ) : ''), $bug, (in_array($bug, $bugsWithNotes) ? 'U' : 'N'), $_BASE_URL, diff --git a/www/search.php b/www/search.php index c2f0ddc..819a0e7 100644 --- a/www/search.php +++ b/www/search.php @@ -1,6 +1,7 @@ Date: Wed, 26 Sep 2018 14:56:46 +0000 Subject: [PATCH 088/141] Fix note updating --- www/dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index c7cf220..1a0616c 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -13,7 +13,7 @@ // handle note and tag updates // -$stmt = $_DB->prepare( 'INSERT INTO metadata (bug, note) VALUES (?, ?) ON DUPLICATE KEY UPDATE note=VALUES(note)' ); +$stmt = $_DB->prepare( 'INSERT INTO metadata (bug, note, stamp) VALUES (?, ?, NOW()) ON DUPLICATE KEY UPDATE note=VALUES(note)' ); if ($_DB->errno) fail( 'Error preparing metadata insert: ' . $_DB->error ); foreach ($_POST AS $key => $value) { if (strncmp( $key, 'note', 4 ) == 0) { From ea4decb1dc228e44495c431178ea47d022ecdebf Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 9 Oct 2018 10:11:31 +0000 Subject: [PATCH 089/141] Start displaying gh_issues table contents; notes aren't hooked up yet --- www/dashboard.php | 55 ++++++++++++++++++++++++++++++++++++----------- www/wipe.php | 3 +++ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/www/dashboard.php b/www/dashboard.php index 1a0616c..049b13c 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -3,6 +3,7 @@ include_once( 'common.php' ); date_default_timezone_set( 'UTC' ); +$_GH_BASE_URL = "https://github.com/"; $_DB = new mysqli( $_MYSQL_HOST, $_MYSQL_USER, $_MYSQL_PASS, $_MYSQL_DB ); if (mysqli_connect_errno()) { @@ -112,6 +113,10 @@ function linkify( $text, $bug ) { return $text; } +function linkify_gh( $text ) { + return $text; +} + function buglinkify( $field, $text ) { global $_BASE_URL; if ($field === 'Depends on' || $field === 'Blocks') { @@ -295,34 +300,58 @@ function safeGet( &$array, $index ) { $reasons[ $row['bug'] ][] = $row['reason']; } +$result = loadTable( 'gh_issues' ); +while ($row = $result->fetch_assoc()) { + $numRows++; + $stamp = strtotime( $row['stamp'] ); + $bugid = $row['repo'] . '#' . $row['issue']; + initEmpty( $bblocks, $bugid, $stamp ); + $bblocks[ $bugid ][ $stamp ] .= sprintf( '
%s said:
%s
', + $row['id'], + $row['author'], + $_GH_BASE_URL, + $row['repo'], + $row['issue'], + $row['commentnum'], + linkify_gh( escapeHTML( $row['comment'] ) ) ) . "\n"; + $reasons[ $bugid ][] = $row['reason']; +} + foreach ($bblocks AS $bug => &$block) { ksort( $block, SORT_NUMERIC ); $touchTime = key( $block ); - $block = sprintf( '
' + if (strpos( $bug, '#' ) !== FALSE) { + $type_gh = true; + $identifier = 'gh_' . $bug; + } else { + $type_gh = false; + $identifier = 'bug' . $bug; + } + $block = sprintf( '
' . 'X ' . '%s' - . 'Bug %d %s' + . '%s %s' . '
' . '
%s
' . '
' - . 'X ' - . 'Back to top' + . 'X ' + . 'Back to top' . '
' . '
', (empty( $meta_secure[ $bug ] ) ? '' : 'secure '), - $bug, + $identifier, (in_array($bug, $bugsWithNotes) ? escapeHTML( safeGet( $meta_notes, $bug ) . ' | ' . safeGet( $meta_tags, $bug ) ) : ''), $bug, (in_array($bug, $bugsWithNotes) ? 'U' : 'N'), - $_BASE_URL, - $bug, - $bug, - escapeHTML( safeGet( $meta_titles, $bug ) ), + $type_gh ? ($_GH_BASE_URL . str_replace( '#', '/issues/', $bug )) + : ($_BASE_URL . '/show_bug.cgi?id=' . $bug), + $type_gh ? $bug : 'Bug ' . $bug, + $type_gh ? '' : escapeHTML( safeGet( $meta_titles, $bug ) ), implode( "\n", $block ), - $bug, - $bug, - $bug, - $bug ) . "\n"; + $identifier, + $identifier, + $identifier, + $identifier ) . "\n"; $col = column( $reasons[ $bug ] ); initEmpty( $columns, $col, $touchTime ); $columns[ $col ][ $touchTime ] .= $block; diff --git a/www/wipe.php b/www/wipe.php index e599b43..cb6d995 100644 --- a/www/wipe.php +++ b/www/wipe.php @@ -29,6 +29,9 @@ case 'c': $table = 'comments'; break; + case 'g': + $table = 'gh_issues'; + break; } $rowId = intval( substr( $id, 1 ) ); $_DB->query( "UPDATE {$table} SET viewed=1 WHERE id={$rowId}" ); From a3403b5f8ff040ddd6d8659bde2d6807465bf4f3 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 9 Oct 2018 10:28:01 +0000 Subject: [PATCH 090/141] Add new GH issues linking --- scraper/src/main.rs | 19 +++++++++++++------ www/dashboard.php | 27 +++++++++++++++++++-------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 5b47a56..1ea44a4 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -55,19 +55,25 @@ fn get_plain_body(mail: &ParsedMail) -> Result, MailParseError> { Ok(None) } -fn url_parts(footer: String) -> Option<(String, String, String)> { +fn url_parts(footer: String) -> Option<(String, String, Option)> { let prefix = "https://github.com/"; let issues = "/issues/"; + let pulls = "/pulls/"; let comment = "#issuecomment-"; let repo_ix = footer.find(prefix)? + prefix.len(); - let issues_ix = footer[repo_ix..].find(issues)? + repo_ix; - let comment_ix = footer[issues_ix..].find(comment)? + issues_ix; - let end_ix = footer[comment_ix..].find("\r\n")? + comment_ix; + let issues_ix = footer[repo_ix..].find(issues).or_else(|| footer[repo_ix..].find(pulls))? + repo_ix; + let issues_len = if footer[issues_ix..].starts_with(issues) { issues.len() } else { pulls.len() }; + let comment_ix = footer[issues_ix..].find(comment).map(|ix| ix + issues_ix); + let end_ix = footer[repo_ix..].find("\r\n")? + repo_ix; + let comment_num = match comment_ix { + Some(ix) => Some(String::from(&footer[ix + comment.len()..end_ix])), + None => None, + }; return Some((String::from(&footer[repo_ix..issues_ix]), - String::from(&footer[issues_ix + issues.len()..comment_ix]), - String::from(&footer[comment_ix + comment.len()..end_ix]))); + String::from(&footer[issues_ix + issues_len..comment_ix.unwrap_or(end_ix)]), + comment_num)); } fn split_footer(msg: String) -> (String, Option) { @@ -91,6 +97,7 @@ fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { }; let footer = footer.ok_or("Unable to find footer".to_string())?; let (repo, issue, commentnum) = url_parts(footer).ok_or("Unable to extract URL parts".to_string())?; + let commentnum = commentnum.unwrap_or(String::from("0")); let db = get_db()?; let result = db.prep_exec(r"INSERT INTO gh_issues (repo, issue, stamp, reason, commentnum, author, comment) diff --git a/www/dashboard.php b/www/dashboard.php index 049b13c..06c91a6 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -306,14 +306,25 @@ function safeGet( &$array, $index ) { $stamp = strtotime( $row['stamp'] ); $bugid = $row['repo'] . '#' . $row['issue']; initEmpty( $bblocks, $bugid, $stamp ); - $bblocks[ $bugid ][ $stamp ] .= sprintf( '
%s said:
%s
', - $row['id'], - $row['author'], - $_GH_BASE_URL, - $row['repo'], - $row['issue'], - $row['commentnum'], - linkify_gh( escapeHTML( $row['comment'] ) ) ) . "\n"; + if ($row['commentnum'] == 0) { + $bblocks[ $bugid ][ $stamp ] .= sprintf( '
New: %s by %s
%s
', + $row['id'], + $_GH_BASE_URL, + $row['repo'], + $row['issue'], + $bugid, + escapeHTML( $row['author'] ), + linkify_gh( escapeHTML( $row['comment'] ), $row['repo'] ) ) . "\n"; + } else { + $bblocks[ $bugid ][ $stamp ] .= sprintf( '
%s said:
%s
', + $row['id'], + escapeHTML( $row['author'] ), + $_GH_BASE_URL, + $row['repo'], + $row['issue'], + $row['commentnum'], + linkify_gh( escapeHTML( $row['comment'] ), $row['repo'] ) ) . "\n"; + } $reasons[ $bugid ][] = $row['reason']; } From f1fb8635c3022f8c9138192d4ec902e3794ac70b Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 9 Oct 2018 10:29:53 +0000 Subject: [PATCH 091/141] Linkify things in GH --- www/dashboard.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index 06c91a6..9887cf8 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -113,7 +113,11 @@ function linkify( $text, $bug ) { return $text; } -function linkify_gh( $text ) { +function linkify_gh( $text, $base_repo ) { + global $_GH_BASE_URL; + $text = preg_replace( '#(https?://\S+)#i', '$1', $text ); + $text = preg_replace( '@(\W)(\w+/\w+)#(\d+)(\W)@', '$1$2#$3$4', $text ); + $text = preg_replace( '@(\W)#(\d+)(\W)@', '$1#$2$3', $text ); return $text; } From 78236e9f19c7cd7aa70eb3e4aeb949c82ba8edfa Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 9 Oct 2018 16:02:20 +0000 Subject: [PATCH 092/141] Expand commentnum to hash since apparently github can send #event- hashes as well as #issuecomment- --- schemas/tables.sql | 2 +- scraper/src/main.rs | 32 ++++++++++++++++---------------- www/dashboard.php | 6 +++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/schemas/tables.sql b/schemas/tables.sql index 4679d10..6aad049 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -83,7 +83,7 @@ CREATE TABLE `gh_issues` ( `stamp` datetime NOT NULL, `viewed` tinyint(1) NOT NULL DEFAULT 0, `reason` varchar(10) NOT NULL, - `commentnum` int(10) NOT NULL, + `hash` varchar(255) NOT NULL, `author` varchar(255) NOT NULL, `comment` mediumtext NOT NULL, PRIMARY KEY (`id`), diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 1ea44a4..6e07e1d 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -58,26 +58,26 @@ fn get_plain_body(mail: &ParsedMail) -> Result, MailParseError> { fn url_parts(footer: String) -> Option<(String, String, Option)> { let prefix = "https://github.com/"; let issues = "/issues/"; - let pulls = "/pulls/"; - let comment = "#issuecomment-"; + let pull = "/pull/"; + let hash = "#"; let repo_ix = footer.find(prefix)? + prefix.len(); - let issues_ix = footer[repo_ix..].find(issues).or_else(|| footer[repo_ix..].find(pulls))? + repo_ix; - let issues_len = if footer[issues_ix..].starts_with(issues) { issues.len() } else { pulls.len() }; - let comment_ix = footer[issues_ix..].find(comment).map(|ix| ix + issues_ix); - let end_ix = footer[repo_ix..].find("\r\n")? + repo_ix; - let comment_num = match comment_ix { - Some(ix) => Some(String::from(&footer[ix + comment.len()..end_ix])), + let issues_ix = footer[repo_ix..].find(issues).or_else(|| footer[repo_ix..].find(pull))? + repo_ix; + let issues_len = if footer[issues_ix..].starts_with(issues) { issues.len() } else { pull.len() }; + let hash_ix = footer[issues_ix..].find(hash).map(|ix| ix + issues_ix); + let end_ix = footer[repo_ix..].find("\n")? + repo_ix; + let hash = match hash_ix { + Some(ix) => Some(String::from(&footer[ix + hash.len()..end_ix])), None => None, }; return Some((String::from(&footer[repo_ix..issues_ix]), - String::from(&footer[issues_ix + issues_len..comment_ix.unwrap_or(end_ix)]), - comment_num)); + String::from(&footer[issues_ix + issues_len..hash_ix.unwrap_or(end_ix)]), + hash)); } fn split_footer(msg: String) -> (String, Option) { - match msg.find("\r\n-- \r\nYou are receiving this because") { + match msg.find("\n-- \nYou are receiving this because") { Some(ix) => (String::from(&msg[0..ix]), Some(String::from(&msg[ix..]))), None => (msg, None), } @@ -96,17 +96,17 @@ fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { None => return Err("No plaintext body found".to_string()), }; let footer = footer.ok_or("Unable to find footer".to_string())?; - let (repo, issue, commentnum) = url_parts(footer).ok_or("Unable to extract URL parts".to_string())?; - let commentnum = commentnum.unwrap_or(String::from("0")); + let (repo, issue, hash) = url_parts(footer).ok_or("Unable to extract URL parts".to_string())?; + let hash = hash.unwrap_or(String::from("")); let db = get_db()?; - let result = db.prep_exec(r"INSERT INTO gh_issues (repo, issue, stamp, reason, commentnum, author, comment) - VALUES (:repo, :issue, FROM_UNIXTIME(:stamp), :reason, :commentnum, :sender, :comment)", params! { + let result = db.prep_exec(r"INSERT INTO gh_issues (repo, issue, stamp, reason, hash, author, comment) + VALUES (:repo, :issue, FROM_UNIXTIME(:stamp), :reason, :hash, :sender, :comment)", params! { repo, issue, stamp, reason, - commentnum, + hash, sender, comment, }).map_err(|e| format!("{:?}", e))?; diff --git a/www/dashboard.php b/www/dashboard.php index 9887cf8..cb386df 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -310,7 +310,7 @@ function safeGet( &$array, $index ) { $stamp = strtotime( $row['stamp'] ); $bugid = $row['repo'] . '#' . $row['issue']; initEmpty( $bblocks, $bugid, $stamp ); - if ($row['commentnum'] == 0) { + if ($row['hash'] == "") { $bblocks[ $bugid ][ $stamp ] .= sprintf( '
New: %s by %s
%s
', $row['id'], $_GH_BASE_URL, @@ -320,13 +320,13 @@ function safeGet( &$array, $index ) { escapeHTML( $row['author'] ), linkify_gh( escapeHTML( $row['comment'] ), $row['repo'] ) ) . "\n"; } else { - $bblocks[ $bugid ][ $stamp ] .= sprintf( '
%s said:
%s
', + $bblocks[ $bugid ][ $stamp ] .= sprintf( '
%s said:
%s
', $row['id'], escapeHTML( $row['author'] ), $_GH_BASE_URL, $row['repo'], $row['issue'], - $row['commentnum'], + $row['hash'], linkify_gh( escapeHTML( $row['comment'] ), $row['repo'] ) ) . "\n"; } $reasons[ $bugid ][] = $row['reason']; From ac589d585374e15a64d826b837370259d51672ac Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 9 Oct 2018 16:02:43 +0000 Subject: [PATCH 093/141] Robustify the scraper against early failure by saving the file and then removing it on success --- scraper/src/main.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 6e07e1d..2159164 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -4,6 +4,7 @@ extern crate mysql; use std::collections::hash_map::DefaultHasher; use std::env; +use std::fs; use std::fs::File; use std::hash::Hasher; use std::io::{Read, stdin, Write}; @@ -12,7 +13,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use mailparse::{dateparse, MailHeaderMap, MailParseError, ParsedMail}; -fn fail(data: &[u8], msg: &str, err: String) -> ! { +fn save_file(data: &[u8]) -> String { let time = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); let mut hasher = DefaultHasher::new(); hasher.write(data); @@ -21,7 +22,11 @@ fn fail(data: &[u8], msg: &str, err: String) -> ! { let mut file = File::create(&uniq_name).unwrap(); file.write(data).ok(); } - let err_name = format!("{}.err", uniq_name); + uniq_name +} + +fn fail(data_file: &str, msg: &str, err: String) -> ! { + let err_name = format!("{}.err", data_file); { let mut file = File::create(err_name).unwrap(); writeln!(file, "{}", msg).ok(); @@ -122,13 +127,15 @@ fn main() { let stdin = stdin(); let mut handle = stdin.lock(); let len = handle.read_to_end(&mut input); - len.err().map(|e| fail(&input, "Reading stdin failed", format!("{:?}", e))); + len.err().map(|e| fail("stdin-read", "Reading stdin failed", format!("{:?}", e))); } - let mail = mailparse::parse_mail(&input).unwrap_or_else(|e| fail(&input, "Unable to parse mail", format!("{:?}", e))); + let saved_file = save_file(&input); + let mail = mailparse::parse_mail(&input).unwrap_or_else(|e| fail(&saved_file, "Unable to parse mail", format!("{:?}", e))); - let github_reason = mail.headers.get_first_value("X-GitHub-Reason").unwrap_or_else(|e| fail(&input, "Unable to read mail header", format!("{:?}", e))); + let github_reason = mail.headers.get_first_value("X-GitHub-Reason").unwrap_or_else(|e| fail(&saved_file, "Unable to read mail header", format!("{:?}", e))); if github_reason.is_some() { - return scrape_github_mail(&mail).unwrap_or_else(|e| fail(&input, "Error while scraping mail", e)); + scrape_github_mail(&mail).unwrap_or_else(|e| fail(&saved_file, "Error while scraping mail", e)); + fs::remove_file(&saved_file).unwrap_or_else(|e| fail(&saved_file, "Error removing file after processing", format!("{:?}", e))); } } From a0e5786132fbfb9c7091e35beb808a8bd4241141 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 15 Oct 2018 19:26:13 +0000 Subject: [PATCH 094/141] Do a better job with newlines --- scraper/src/main.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 2159164..f1050f9 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -70,7 +70,7 @@ fn url_parts(footer: String) -> Option<(String, String, Option)> { let issues_ix = footer[repo_ix..].find(issues).or_else(|| footer[repo_ix..].find(pull))? + repo_ix; let issues_len = if footer[issues_ix..].starts_with(issues) { issues.len() } else { pull.len() }; let hash_ix = footer[issues_ix..].find(hash).map(|ix| ix + issues_ix); - let end_ix = footer[repo_ix..].find("\n")? + repo_ix; + let end_ix = footer[repo_ix..].find("\n").map(|ix| ix + repo_ix).unwrap_or(footer.len()); let hash = match hash_ix { Some(ix) => Some(String::from(&footer[ix + hash.len()..end_ix])), None => None, @@ -82,7 +82,10 @@ fn url_parts(footer: String) -> Option<(String, String, Option)> { } fn split_footer(msg: String) -> (String, Option) { - match msg.find("\n-- \nYou are receiving this because") { + // Avoid trying to match newlines directly since they can be either \r\n or \n + let ix = msg.find("You are receiving this because") + .and_then(|ix| msg[0..ix].rfind("-- ")); + match ix { Some(ix) => (String::from(&msg[0..ix]), Some(String::from(&msg[ix..]))), None => (msg, None), } From 59ab00ddbff8c440d375885f68ddfcb82f30d01b Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 16 Oct 2018 12:02:08 +0000 Subject: [PATCH 095/141] Deal with URLs that have no hash but /files/.. instead --- scraper/src/main.rs | 8 +++++--- www/dashboard.php | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index f1050f9..a07df52 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -65,19 +65,21 @@ fn url_parts(footer: String) -> Option<(String, String, Option)> { let issues = "/issues/"; let pull = "/pull/"; let hash = "#"; + let slash = "/"; let repo_ix = footer.find(prefix)? + prefix.len(); let issues_ix = footer[repo_ix..].find(issues).or_else(|| footer[repo_ix..].find(pull))? + repo_ix; let issues_len = if footer[issues_ix..].starts_with(issues) { issues.len() } else { pull.len() }; - let hash_ix = footer[issues_ix..].find(hash).map(|ix| ix + issues_ix); + let issuenum_ix = issues_ix + issues_len; + let hash_ix = footer[issuenum_ix..].find(hash).or_else(|| footer[issuenum_ix..].find(slash)).map(|ix| ix + issuenum_ix); let end_ix = footer[repo_ix..].find("\n").map(|ix| ix + repo_ix).unwrap_or(footer.len()); let hash = match hash_ix { - Some(ix) => Some(String::from(&footer[ix + hash.len()..end_ix])), + Some(ix) => Some(String::from(&footer[ix..end_ix])), None => None, }; return Some((String::from(&footer[repo_ix..issues_ix]), - String::from(&footer[issues_ix + issues_len..hash_ix.unwrap_or(end_ix)]), + String::from(&footer[issuenum_ix..hash_ix.unwrap_or(end_ix)]), hash)); } diff --git a/www/dashboard.php b/www/dashboard.php index cb386df..a561494 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -320,11 +320,12 @@ function safeGet( &$array, $index ) { escapeHTML( $row['author'] ), linkify_gh( escapeHTML( $row['comment'] ), $row['repo'] ) ) . "\n"; } else { - $bblocks[ $bugid ][ $stamp ] .= sprintf( '
%s said:
%s
', + $bblocks[ $bugid ][ $stamp ] .= sprintf( '
%s said:
%s
', $row['id'], escapeHTML( $row['author'] ), $_GH_BASE_URL, $row['repo'], + (strncmp( $row['hash'], '#', 1 ) == 0 ? 'issues' : 'pull'), $row['issue'], $row['hash'], linkify_gh( escapeHTML( $row['comment'] ), $row['repo'] ) ) . "\n"; From 1c1e59fd777efd821a5d8d24fa1bc75cb54accde Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 16 Oct 2018 12:16:24 +0000 Subject: [PATCH 096/141] Alter schema to support non-BMP unicode --- schemas/tables.sql | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/schemas/tables.sql b/schemas/tables.sql index 6aad049..1282ad3 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -11,7 +11,7 @@ CREATE TABLE `requests` ( KEY (`bug`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `reviews` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -29,7 +29,7 @@ CREATE TABLE `reviews` ( KEY (`bug`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `changes` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -44,7 +44,7 @@ CREATE TABLE `changes` ( KEY (`bug`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `comments` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -59,7 +59,7 @@ CREATE TABLE `comments` ( KEY (`bug`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `newbugs` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -74,11 +74,11 @@ CREATE TABLE `newbugs` ( KEY (`bug`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `gh_issues` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `repo` varchar(255) NOT NULL, + `repo` varchar(100) NOT NULL, `issue` int(10) unsigned NOT NULL, `stamp` datetime NOT NULL, `viewed` tinyint(1) NOT NULL DEFAULT 0, @@ -90,7 +90,7 @@ CREATE TABLE `gh_issues` ( KEY (`repo`, `issue`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `metadata` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -101,13 +101,13 @@ CREATE TABLE `metadata` ( `note` mediumtext NOT NULL DEFAULT '', PRIMARY KEY (`id`), UNIQUE KEY (`bug`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `tags` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `bug` int(10) unsigned NOT NULL, - `tag` varchar(255) NOT NULL, + `tag` varchar(100) NOT NULL, PRIMARY KEY (`id`), KEY (`bug`), KEY (`tag`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; From d2c072b3c309a9d2d8824c5a7cc55a4c40a77e48 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 16 Oct 2018 15:59:40 +0000 Subject: [PATCH 097/141] Map thread authoring to reporter --- scraper/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index a07df52..ea98ab2 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -97,6 +97,7 @@ fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { let sender = mail.headers.get_first_value("X-GitHub-Sender").map_err(|e| format!("{:?}", e))?.unwrap_or("Unknown".to_string()); let reason = match mail.headers.get_first_value("X-GitHub-Reason").map_err(|e| format!("{:?}", e))? { Some(ref s) if s == "review_requested" => "review", + Some(ref s) if s == "author" => "Reporter", Some(ref s) if s == "mention" => "CC", _ => "Watch", }; From 424d7f60b851d1045c03e5a0c2da607efdc82c22 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 18 Oct 2018 00:30:13 +0000 Subject: [PATCH 098/141] Make some columns wider --- schemas/tables.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schemas/tables.sql b/schemas/tables.sql index 1282ad3..5da2d29 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -38,8 +38,8 @@ CREATE TABLE `changes` ( `viewed` tinyint(1) NOT NULL DEFAULT 0, `reason` varchar(10) NOT NULL, `field` varchar(255) NOT NULL, - `oldval` varchar(255) NOT NULL, - `newval` varchar(255) NOT NULL, + `oldval` varchar(1024) NOT NULL, + `newval` varchar(1024) NOT NULL, PRIMARY KEY (`id`), KEY (`bug`), KEY (`stamp`), @@ -96,7 +96,7 @@ CREATE TABLE `metadata` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `bug` int(10) unsigned NOT NULL DEFAULT 0, `stamp` datetime NOT NULL, - `title` varchar(255) NOT NULL DEFAULT '', + `title` varchar(1024) NOT NULL DEFAULT '', `secure` tinyint(1) NOT NULL DEFAULT 0, `note` mediumtext NOT NULL DEFAULT '', PRIMARY KEY (`id`), From 66de671d5faf9f2bb60da719ed38ec7f3e7e691c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 18 Oct 2018 00:42:32 +0000 Subject: [PATCH 099/141] Switch metadata bug to varchar to allow GH issues as well --- schemas/tables.sql | 2 +- scraper/filter.php | 2 +- www/dashboard.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/schemas/tables.sql b/schemas/tables.sql index 5da2d29..866c504 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -94,7 +94,7 @@ CREATE TABLE `gh_issues` ( CREATE TABLE `metadata` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `bug` int(10) unsigned NOT NULL DEFAULT 0, + `bug` varchar(128) unsigned NOT NULL, `stamp` datetime NOT NULL, `title` varchar(1024) NOT NULL DEFAULT '', `secure` tinyint(1) NOT NULL DEFAULT 0, diff --git a/scraper/filter.php b/scraper/filter.php index 655b385..ff83631 100755 --- a/scraper/filter.php +++ b/scraper/filter.php @@ -406,7 +406,7 @@ function updateMetadata( $date ) { $stmt = prepare( 'INSERT INTO metadata (bug, stamp, title, secure, note) VALUES (?, ?, ?, ?, ?) ' . 'ON DUPLICATE KEY UPDATE stamp=VALUES(stamp), title=VALUES(title), secure=VALUES(secure)' ); $note = ""; - if (!$stmt->bind_param( 'issis', $matches[1], $date, $matches[2], $bugIsSecure, $note )) { + if (!$stmt->bind_param( 'sssis', $matches[1], $date, $matches[2], $bugIsSecure, $note )) { fail( "Binding params failed for metadata: [{$stmt->error}]" ); } if (!$stmt->execute()) { diff --git a/www/dashboard.php b/www/dashboard.php index a561494..874c91b 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -18,7 +18,7 @@ if ($_DB->errno) fail( 'Error preparing metadata insert: ' . $_DB->error ); foreach ($_POST AS $key => $value) { if (strncmp( $key, 'note', 4 ) == 0) { - $stmt->bind_param( 'is', intval( substr( $key, 4 ) ), trim( $value ) ); + $stmt->bind_param( 'ss', substr( $key, 4 ), trim( $value ) ); $stmt->execute(); } } From b54aec52a397bd9bd0c99cf2afd8fdf9aa63a67b Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 18 Oct 2018 01:15:35 +0000 Subject: [PATCH 100/141] Remove stray \r characters that might be just before end_ix --- scraper/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index ea98ab2..74e43a9 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -74,12 +74,12 @@ fn url_parts(footer: String) -> Option<(String, String, Option)> { let hash_ix = footer[issuenum_ix..].find(hash).or_else(|| footer[issuenum_ix..].find(slash)).map(|ix| ix + issuenum_ix); let end_ix = footer[repo_ix..].find("\n").map(|ix| ix + repo_ix).unwrap_or(footer.len()); let hash = match hash_ix { - Some(ix) => Some(String::from(&footer[ix..end_ix])), + Some(ix) => Some(String::from(footer[ix..end_ix].trim())), None => None, }; return Some((String::from(&footer[repo_ix..issues_ix]), - String::from(&footer[issuenum_ix..hash_ix.unwrap_or(end_ix)]), + String::from(footer[issuenum_ix..hash_ix.unwrap_or(end_ix)].trim()), hash)); } From 3b97730f7bf3d973814c07d5f742289f8de512c6 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 18 Oct 2018 01:15:45 +0000 Subject: [PATCH 101/141] Add title support for GH issues --- scraper/src/main.rs | 29 ++++++++++++++++++++++++++--- www/dashboard.php | 4 ++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 74e43a9..074cdb6 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -94,6 +94,19 @@ fn split_footer(msg: String) -> (String, Option) { } fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { + let mut title = mail.headers.get_first_value("Subject").map_err(|e| format!("{:?}", e))?.unwrap_or("(no subject)".to_string()); + title = title.trim_left_matches("Re: ").to_string(); + if title.starts_with("[") { + if let Some(close_bracket) = title.find("]") { + title = title[close_bracket + 1..].trim().to_string(); + } + } + if title.ends_with(")") { + if let Some(issue_start) = title.rfind("(#") { + title = title[0..issue_start].trim().to_string(); + } + } + let sender = mail.headers.get_first_value("X-GitHub-Sender").map_err(|e| format!("{:?}", e))?.unwrap_or("Unknown".to_string()); let reason = match mail.headers.get_first_value("X-GitHub-Reason").map_err(|e| format!("{:?}", e))? { Some(ref s) if s == "review_requested" => "review", @@ -110,9 +123,18 @@ fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { let (repo, issue, hash) = url_parts(footer).ok_or("Unable to extract URL parts".to_string())?; let hash = hash.unwrap_or(String::from("")); + let id = format!("{}#{}", repo, issue); let db = get_db()?; - let result = db.prep_exec(r"INSERT INTO gh_issues (repo, issue, stamp, reason, hash, author, comment) - VALUES (:repo, :issue, FROM_UNIXTIME(:stamp), :reason, :hash, :sender, :comment)", params! { + db.prep_exec(r#"INSERT INTO metadata (bug, stamp, title, secure, note) + VALUES (:id, FROM_UNIXTIME(:stamp), :title, 0, "") + ON DUPLICATE KEY UPDATE stamp=VALUES(stamp), title=VALUES(title)"#, params! { + id, + stamp, + title, + }).map_err(|e| format!("{:?}", e))?; + + let result = db.prep_exec(r#"INSERT INTO gh_issues (repo, issue, stamp, reason, hash, author, comment) + VALUES (:repo, :issue, FROM_UNIXTIME(:stamp), :reason, :hash, :sender, :comment)"#, params! { repo, issue, stamp, @@ -122,8 +144,9 @@ fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { comment, }).map_err(|e| format!("{:?}", e))?; if result.affected_rows() != 1 { - return Err(format!("Affected row count was {}, not 1", result.affected_rows())); + return Err(format!("Affected row count for gh_issues was {}, not 1", result.affected_rows())); } + Ok(()) } diff --git a/www/dashboard.php b/www/dashboard.php index 874c91b..a8822be 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -316,7 +316,7 @@ function safeGet( &$array, $index ) { $_GH_BASE_URL, $row['repo'], $row['issue'], - $bugid, + escapeHTML( safeGet( $meta_titles, $bugid ) ), escapeHTML( $row['author'] ), linkify_gh( escapeHTML( $row['comment'] ), $row['repo'] ) ) . "\n"; } else { @@ -362,7 +362,7 @@ function safeGet( &$array, $index ) { $type_gh ? ($_GH_BASE_URL . str_replace( '#', '/issues/', $bug )) : ($_BASE_URL . '/show_bug.cgi?id=' . $bug), $type_gh ? $bug : 'Bug ' . $bug, - $type_gh ? '' : escapeHTML( safeGet( $meta_titles, $bug ) ), + escapeHTML( safeGet( $meta_titles, $bug ) ), implode( "\n", $block ), $identifier, $identifier, From 3706b1303fe1c8c259bf69c8ec33e2f037ea61d0 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 18 Oct 2018 03:09:50 +0000 Subject: [PATCH 102/141] Linkify markdown --- www/dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index a8822be..4aed343 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -115,7 +115,7 @@ function linkify( $text, $bug ) { function linkify_gh( $text, $base_repo ) { global $_GH_BASE_URL; - $text = preg_replace( '#(https?://\S+)#i', '$1', $text ); + $text = preg_replace( '#\[(.*?)\]\((https?://.*?)\)#i', '$1', $text ); // markdown links $text = preg_replace( '@(\W)(\w+/\w+)#(\d+)(\W)@', '$1$2#$3$4', $text ); $text = preg_replace( '@(\W)#(\d+)(\W)@', '$1#$2$3', $text ); return $text; From f4d52aa8f8f360fead0bd073275cb7148a6119a3 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 18 Oct 2018 03:49:35 +0000 Subject: [PATCH 103/141] Fix typo --- schemas/tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/tables.sql b/schemas/tables.sql index 866c504..259f569 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -94,7 +94,7 @@ CREATE TABLE `gh_issues` ( CREATE TABLE `metadata` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `bug` varchar(128) unsigned NOT NULL, + `bug` varchar(128) NOT NULL, `stamp` datetime NOT NULL, `title` varchar(1024) NOT NULL DEFAULT '', `secure` tinyint(1) NOT NULL DEFAULT 0, From 40c9ffdebaf567138201e905e5adb92633cb1525 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 18 Oct 2018 03:49:58 +0000 Subject: [PATCH 104/141] Fix note/tag stuff for GH issues --- schemas/tables.sql | 2 +- www/common.php | 7 ++-- www/dashboard.php | 81 ++++++++++++++++++++++++++++------------------ 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/schemas/tables.sql b/schemas/tables.sql index 259f569..cade78f 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -105,7 +105,7 @@ CREATE TABLE `metadata` ( CREATE TABLE `tags` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `bug` int(10) unsigned NOT NULL, + `bug` varchar(128) NOT NULL, `tag` varchar(100) NOT NULL, PRIMARY KEY (`id`), KEY (`bug`), diff --git a/www/common.php b/www/common.php index 2fc5eaf..1de4126 100644 --- a/www/common.php +++ b/www/common.php @@ -13,6 +13,7 @@ } function fail( $message ) { + error_log( $message ); header( 'HTTP/500 Error!' ); print $message; exit( 0 ); @@ -27,8 +28,9 @@ function updateTags( $newTags ) { $stmt = $_DB->prepare( 'DELETE FROM tags WHERE bug=?' ); if ($_DB->errno) fail( 'Error preparing tag deletion: ' . $_DB->error ); foreach ($newTags AS $bug => $tagList) { - $stmt->bind_param( 'i', $bug ); + $stmt->bind_param( 's', $bug ); $stmt->execute(); + if ($stmt->errno) fail( 'Error inserting to metadata: ' . $stmt->error ); } $stmt->close(); @@ -38,8 +40,9 @@ function updateTags( $newTags ) { foreach (explode( ',', $tagList ) AS $tag) { $tag = trim( $tag ); if (strlen( $tag ) > 0) { - $stmt->bind_param( 'is', $bug, $tag ); + $stmt->bind_param( 'ss', $bug, $tag ); $stmt->execute(); + if ($stmt->errno) fail( 'Error inserting to metadata: ' . $stmt->error ); } } } diff --git a/www/dashboard.php b/www/dashboard.php index 4aed343..f42e268 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -20,6 +20,7 @@ if (strncmp( $key, 'note', 4 ) == 0) { $stmt->bind_param( 'ss', substr( $key, 4 ), trim( $value ) ); $stmt->execute(); + if ($stmt->errno) fail( 'Error inserting to metadata: ' . $stmt->error ); } } $stmt->close(); @@ -27,7 +28,7 @@ $tagUpdates = array(); foreach ($_POST AS $key => $value) { if (strncmp( $key, 'tags', 4 ) == 0) { - $tagUpdates[ intval( substr( $key, 4 ) ) ] = $value; + $tagUpdates[ substr( $key, 4 ) ] = $value; } } updateTags( $tagUpdates ); @@ -345,7 +346,7 @@ function safeGet( &$array, $index ) { } $block = sprintf( '
' . 'X ' - . '%s' + . '%s' . '%s %s' . '
' . '
%s
' @@ -504,13 +505,14 @@ function wipe(e) { } }, true ); - function addNote( bugnumber ) { + function addNote( bugid ) { var notediv = document.createElement( "div" ); notediv.className = "newnote"; var sibling = document.getElementById( "notebuttons" ); sibling.parentNode.insertBefore( notediv, sibling ); - notediv.innerHTML = 'Bug : '; - if (bugnumber) { + var prefix = bugid.includes( '#' ) ? '' : 'Bug '; + notediv.innerHTML = '' + prefix + ': '; + if (bugid) { notediv.getElementsByTagName( "input" )[1].focus(); } else { notediv.getElementsByTagName( "input" )[0].focus(); @@ -522,29 +524,39 @@ function setNoteNames() { while (newnotes.length > 0) { var newnote = newnotes[0]; var bugnumbertext = newnote.getElementsByTagName( "input" )[0].value; - var bugnumber = parseInt( bugnumbertext ); - if (isNaN( bugnumber )) { - if (window.confirm( "Unable to parse " + bugnumbertext + " as a bug number; replace with 0 and continue anyway?" )) { - bugnumber = 0; - } else { - return false; + var bugid; + if (bugnumbertext.includes("#")) { + // GH issue + bugid = bugnumbertext; + var anchor = document.createElement( "a" ); + anchor.setAttribute( "href", "" + bugnumbertext.replace( '#', '/issues' ) ); + anchor.textContent = bugnumbertext; + } else { + // bugzilla bug + bugid = parseInt( bugnumbertext ); + if (isNaN( bugid )) { + if (window.confirm( "Unable to parse " + bugnumbertext + " as a bug number; replace with 0 and continue anyway?" )) { + bugid = 0; + } else { + return false; + } } + var anchor = document.createElement( "a" ); + anchor.setAttribute( "href", "/show_bug.cgi?id=" + bugid ); + anchor.textContent = "Bug " + bugid; } - var anchor = document.createElement( "a" ); - anchor.setAttribute( "href", "/show_bug.cgi?id=" + bugnumber ); - anchor.textContent = "Bug " + bugnumber; newnote.replaceChild( anchor, newnote.getElementsByTagName( "span" )[0] ); - newnote.getElementsByTagName( "input" )[0].setAttribute( "name", "note" + bugnumber ); - newnote.getElementsByTagName( "input" )[1].setAttribute( "name", "tags" + bugnumber ); + newnote.getElementsByTagName( "input" )[0].setAttribute( "name", "note" + bugid ); + newnote.getElementsByTagName( "input" )[1].setAttribute( "name", "tags" + bugid ); newnote.className = "note"; } return true; } - function noteify( linkElement, bugnumber ) { + function noteify( linkElement, bugid ) { var notes = document.getElementsByClassName( "note" ); // see if we can find a note already for this bug and just give it focus - var search = "Bug " + bugnumber; + var search = bugid.includes("#") ? bugid : "Bug " + bugid; for (var i = 0; i < notes.length; i++) { if (notes[i].firstChild.textContent == search) { notes[i].getElementsByTagName( "input" )[0].focus(); @@ -554,13 +566,13 @@ function noteify( linkElement, bugnumber ) { // also search through the newly-added notes that are in a different format notes = document.getElementsByClassName( "newnote" ); for (var i = 0; i < notes.length; i++) { - if (notes[i].getElementsByTagName( "input" )[0].value == bugnumber) { + if (notes[i].getElementsByTagName( "input" )[0].value == bugid) { notes[i].getElementsByTagName( "input" )[1].focus(); return false; } } // couldn't find it, so add a new one - addNote( bugnumber ); + addNote( bugid ); linkElement.textContent = 'U'; return false; } @@ -585,19 +597,24 @@ function noteify( linkElement, bugnumber ) {
Bug notes Bug %d: ' - . '' - . ' ' +foreach ($bugsWithNotes AS $bugid) { + if (strpos( $bugid, '#' ) !== FALSE) { + $type_gh = true; + } else { + $type_gh = false; + } + echo sprintf( '
%s: ' + . '' + . ' ' . '%s
', - $_BASE_URL, - $bug, - $bug, - $bug, - escapeHTML( safeGet( $meta_notes, $bug ) ), - $bug, - escapeHTML( safeGet( $meta_tags, $bug ) ), - escapeHTML( safeGet( $meta_titles, $bug ) ) ), + $type_gh ? ($_GH_BASE_URL . str_replace( '#', '/issues/', $bugid )) + : ($_BASE_URL . '/show_bug.cgi?id=' . $bugid), + $type_gh ? $bugid : "Bug " . $bugid, + $bugid, + escapeHTML( safeGet( $meta_notes, $bugid ) ), + $bugid, + escapeHTML( safeGet( $meta_tags, $bugid ) ), + escapeHTML( safeGet( $meta_titles, $bugid ) ) ), "\n"; } ?> From 48bf5378265bbfc7afc4eb16dfe904ee97578470 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 19 Oct 2018 15:16:30 +0000 Subject: [PATCH 105/141] Increase newbugs.title to 1024 len --- schemas/tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/tables.sql b/schemas/tables.sql index cade78f..c8f3c8d 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -67,7 +67,7 @@ CREATE TABLE `newbugs` ( `stamp` datetime NOT NULL, `viewed` tinyint(1) NOT NULL DEFAULT 0, `reason` varchar(10) NOT NULL, - `title` varchar(255) NOT NULL, + `title` varchar(1024) NOT NULL, `author` varchar(255) NOT NULL, `description` mediumtext NOT NULL, PRIMARY KEY (`id`), From 6945af3bab3ab4f2288a2162c4fe60a76f2be708 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 2 Nov 2018 08:26:40 +0000 Subject: [PATCH 106/141] Factor out some bugzilla vs github code --- www/common.php | 19 +++++++++++++++++++ www/dashboard.php | 26 +++++--------------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/www/common.php b/www/common.php index 1de4126..7dbef56 100644 --- a/www/common.php +++ b/www/common.php @@ -21,6 +21,7 @@ function fail( $message ) { $BUGMASH_DIR = $_SERVER['DOCUMENT_ROOT'] . '/../scraper'; include_once( $BUGMASH_DIR . '/config.php' ); +$_GH_BASE_URL = "https://github.com/"; function updateTags( $newTags ) { global $_DB; @@ -55,4 +56,22 @@ function escapeHTML( $stuff ) { return $stuff; } +function isGithubIssue( $bugid ) { + return (strpos( $bugid, '#' ) !== FALSE); +} + +function makeBugLink( $bugid ) { + global $_BASE_URL, $_GH_BASE_URL; + + if (isGithubIssue( $bugid )) { + return sprintf( '%s', + $_GH_BASE_URL . str_replace( '#', '/issues/', $bugid ), + $bugid); + } else { + return sprintf( '%s', + $_BASE_URL . '/show_bug.cgi?id=' . $bugid, + "Bug " . $bugid); + } +} + ?> diff --git a/www/dashboard.php b/www/dashboard.php index f42e268..2bdf179 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -3,7 +3,6 @@ include_once( 'common.php' ); date_default_timezone_set( 'UTC' ); -$_GH_BASE_URL = "https://github.com/"; $_DB = new mysqli( $_MYSQL_HOST, $_MYSQL_USER, $_MYSQL_PASS, $_MYSQL_DB ); if (mysqli_connect_errno()) { @@ -337,17 +336,11 @@ function safeGet( &$array, $index ) { foreach ($bblocks AS $bug => &$block) { ksort( $block, SORT_NUMERIC ); $touchTime = key( $block ); - if (strpos( $bug, '#' ) !== FALSE) { - $type_gh = true; - $identifier = 'gh_' . $bug; - } else { - $type_gh = false; - $identifier = 'bug' . $bug; - } + $identifier = (isGithubIssue( $bug ) ? 'gh_' : 'bug') . $bug; $block = sprintf( '
' . 'X ' . '%s' - . '%s %s' + . '%s %s' . '
' . '
%s
' . '
' @@ -360,9 +353,7 @@ function safeGet( &$array, $index ) { (in_array($bug, $bugsWithNotes) ? escapeHTML( safeGet( $meta_notes, $bug ) . ' | ' . safeGet( $meta_tags, $bug ) ) : ''), $bug, (in_array($bug, $bugsWithNotes) ? 'U' : 'N'), - $type_gh ? ($_GH_BASE_URL . str_replace( '#', '/issues/', $bug )) - : ($_BASE_URL . '/show_bug.cgi?id=' . $bug), - $type_gh ? $bug : 'Bug ' . $bug, + makeBugLink( $bug ), escapeHTML( safeGet( $meta_titles, $bug ) ), implode( "\n", $block ), $identifier, @@ -598,18 +589,11 @@ function noteify( linkElement, bugid ) { Bug notes %s: ' + echo sprintf( '
%s: ' . '' . ' ' . '%s
', - $type_gh ? ($_GH_BASE_URL . str_replace( '#', '/issues/', $bugid )) - : ($_BASE_URL . '/show_bug.cgi?id=' . $bugid), - $type_gh ? $bugid : "Bug " . $bugid, + makeBugLink( $bugid ), $bugid, escapeHTML( safeGet( $meta_notes, $bugid ) ), $bugid, From 0808eb158e8dcc7a7bd150862ee574ddbd4cc55c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 2 Nov 2018 08:27:31 +0000 Subject: [PATCH 107/141] Fix search results for GH issues --- www/search.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/www/search.php b/www/search.php index 819a0e7..21267dc 100644 --- a/www/search.php +++ b/www/search.php @@ -238,10 +238,8 @@ function formatHits( $text, $terms, $isTitle ) { if (! $formattedTitle) { $formattedTitle = escapeHTML( $meta_titles[ $bug ] ); } - echo sprintf( '
Bug %d %s
', - $_BASE_URL, - $bug, - $bug, + echo sprintf( '
%s %s
', + makeBugLink( $bug ), $formattedTitle ), "\n"; foreach ($hits AS $hit) { echo '
', $hit, '
', "\n"; From cb153155dc87a2720764c77d987d634d47f69591 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 19 Nov 2018 17:56:18 +0000 Subject: [PATCH 108/141] Add new robot name --- www/dashboard.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index 2bdf179..f4f40af 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -290,7 +290,8 @@ function safeGet( &$array, $index ) { $numRows++; $stamp = strtotime( $row['stamp'] ); $isTbplRobot = (stripWhitespace( $row['author'] ) == 'TBPLRobot') - || (stripWhitespace( $row['author'] ) == 'TreeherderRobot'); + || (stripWhitespace( $row['author'] ) == 'TreeherderRobot') + || (stripWhitespace( $row['author'] ) == 'IntermittentFailuresRobot'); initEmpty( $bblocks, $row['bug'], $stamp ); $bblocks[ $row['bug'] ][ $stamp ] .= sprintf( '
%s said:
%s
', ($hide ? 'display: none;' : 'white-space: pre-line;'), From 6c38a8f122997b57b7ac62f5a3e93f55ebdd60da Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 20 Nov 2018 10:28:48 +0000 Subject: [PATCH 109/141] Handle emails generated from replies to other emails --- scraper/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 074cdb6..bd1d290 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -85,7 +85,7 @@ fn url_parts(footer: String) -> Option<(String, String, Option)> { fn split_footer(msg: String) -> (String, Option) { // Avoid trying to match newlines directly since they can be either \r\n or \n - let ix = msg.find("You are receiving this because") + let ix = msg.rfind("You are receiving this because") .and_then(|ix| msg[0..ix].rfind("-- ")); match ix { Some(ix) => (String::from(&msg[0..ix]), Some(String::from(&msg[ix..]))), From b9ebdd092bb51f46347288a0f6fb63b361e0929c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 14 Jan 2019 16:34:38 +0000 Subject: [PATCH 110/141] Support commit notifications --- schemas/tables.sql | 2 +- scraper/src/main.rs | 19 +++++++++++++++---- www/common.php | 7 ++++++- www/dashboard.php | 3 ++- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/schemas/tables.sql b/schemas/tables.sql index c8f3c8d..7959a4b 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -79,7 +79,7 @@ CREATE TABLE `newbugs` ( CREATE TABLE `gh_issues` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `repo` varchar(100) NOT NULL, - `issue` int(10) unsigned NOT NULL, + `issue` varchar(40) NOT NULL, `stamp` datetime NOT NULL, `viewed` tinyint(1) NOT NULL DEFAULT 0, `reason` varchar(10) NOT NULL, diff --git a/scraper/src/main.rs b/scraper/src/main.rs index bd1d290..dd9e25b 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -62,14 +62,25 @@ fn get_plain_body(mail: &ParsedMail) -> Result, MailParseError> { fn url_parts(footer: String) -> Option<(String, String, Option)> { let prefix = "https://github.com/"; - let issues = "/issues/"; - let pull = "/pull/"; + let types = ["/issues/", "/pull/", "/commit/"]; let hash = "#"; let slash = "/"; let repo_ix = footer.find(prefix)? + prefix.len(); - let issues_ix = footer[repo_ix..].find(issues).or_else(|| footer[repo_ix..].find(pull))? + repo_ix; - let issues_len = if footer[issues_ix..].starts_with(issues) { issues.len() } else { pull.len() }; + let mut issues_ix = None; + let mut issues_len = None; + for t in types.iter() { + match footer[repo_ix..].find(t) { + Some(x) => { + issues_ix = Some(x); + issues_len = Some(t.len()); + break; + } + None => continue, + } + } + let issues_ix = issues_ix? + repo_ix; + let issues_len = issues_len.unwrap(); let issuenum_ix = issues_ix + issues_len; let hash_ix = footer[issuenum_ix..].find(hash).or_else(|| footer[issuenum_ix..].find(slash)).map(|ix| ix + issuenum_ix); let end_ix = footer[repo_ix..].find("\n").map(|ix| ix + repo_ix).unwrap_or(footer.len()); diff --git a/www/common.php b/www/common.php index 7dbef56..d9940ff 100644 --- a/www/common.php +++ b/www/common.php @@ -60,12 +60,17 @@ function isGithubIssue( $bugid ) { return (strpos( $bugid, '#' ) !== FALSE); } +function isGithubCommit( $bugid ) { + return (strlen( $bugid ) - strpos( $bugid, '#' )) >= 40; +} + function makeBugLink( $bugid ) { global $_BASE_URL, $_GH_BASE_URL; if (isGithubIssue( $bugid )) { + $type = isGithubCommit( $bugid ) ? '/commit/' : '/issues/'; return sprintf( '%s', - $_GH_BASE_URL . str_replace( '#', '/issues/', $bugid ), + $_GH_BASE_URL . str_replace( '#', $type, $bugid ), $bugid); } else { return sprintf( '%s', diff --git a/www/dashboard.php b/www/dashboard.php index f4f40af..890a60d 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -312,10 +312,11 @@ function safeGet( &$array, $index ) { $bugid = $row['repo'] . '#' . $row['issue']; initEmpty( $bblocks, $bugid, $stamp ); if ($row['hash'] == "") { - $bblocks[ $bugid ][ $stamp ] .= sprintf( '
New: %s by %s
%s
', + $bblocks[ $bugid ][ $stamp ] .= sprintf( '
New: %s by %s
%s
', $row['id'], $_GH_BASE_URL, $row['repo'], + strlen( $row['issue'] ) == 40 ? 'commit' : 'issues', $row['issue'], escapeHTML( safeGet( $meta_titles, $bugid ) ), escapeHTML( $row['author'] ), From e979a73ac65c852de2f88e3fadaee3eadd7d40ba Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 22 Jan 2019 22:35:56 +0000 Subject: [PATCH 111/141] Fix scraping of attachment props change --- scraper/filter.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scraper/filter.php b/scraper/filter.php index ff83631..a986e52 100755 --- a/scraper/filter.php +++ b/scraper/filter.php @@ -242,10 +242,7 @@ function parseChangeTable( $fields, $rows ) { continue; } $matchedStart = false; - if (stripos( $fields[ $ixField ], $col1 ) === 0) { - // simple case match the start of the field against the column - $matchedStart = true; - } else if (preg_match( '/^Attachment/', $col1 )) { + if (preg_match( '/^Attachment/', $col1 )) { // Sometimes the attachment number is on the second line. Make sure we pick it up $col1 .= ' ' . trim( substr( $rows[$i+1], 0, $widths[0] ) ); // Special case for flags, because the changed header just has "Flags" instead of "Attachment Flags" @@ -257,6 +254,9 @@ function parseChangeTable( $fields, $rows ) { $fields[ $ixField ] = preg_replace( '/(Attachment #\d+).*/', '\1', $col1 ) . substr( $fields[ $ixField ], strlen( 'Attachment' )); $matchedStart = true; } + } else if (stripos( $fields[ $ixField ], $col1 ) === 0) { + // simple case match the start of the field against the column + $matchedStart = true; } if ($matchedStart) { if ($ixField > 0) { From efb5905e02df24b32684cf181a6c15f6df0d6f52 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 14 Feb 2019 16:11:52 +0000 Subject: [PATCH 112/141] Add Fission Milestone field --- scraper/filter.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scraper/filter.php b/scraper/filter.php index a986e52..f70d68e 100755 --- a/scraper/filter.php +++ b/scraper/filter.php @@ -177,7 +177,8 @@ function normalizeFieldList( $fieldString ) { || $word == 'Last' /* Resolved */ || $word == 'QA' /* Whiteboard */ || $word == 'Restrict' /* Comments */ - || $word == 'Due' /* Date */) + || $word == 'Due' /* Date */ + || $word == 'Fission' /* Milestone */) { if ($i + 1 >= count( $words )) { fail( 'Unrecognized field list (4): ' . print_r( $words, true ) ); From 7eac6bea82aaf6f91d3407b1d99bd75755140495 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 8 Apr 2019 14:38:32 +0000 Subject: [PATCH 113/141] Link bugs in regressions field --- www/dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index 890a60d..f810065 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -123,7 +123,7 @@ function linkify_gh( $text, $base_repo ) { function buglinkify( $field, $text ) { global $_BASE_URL; - if ($field === 'Depends on' || $field === 'Blocks') { + if ($field === 'Depends on' || $field === 'Blocks' || $field === 'Regressions') { $text = preg_replace( '/(\d+)/e', 'buglink(\'\', \'\\1\')', $text ); } return $text; From 1a8deb632b17c97b708033566ae72567508177e4 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 8 Apr 2019 14:42:46 +0000 Subject: [PATCH 114/141] Add Regressed by field --- scraper/filter.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scraper/filter.php b/scraper/filter.php index f70d68e..b496b19 100755 --- a/scraper/filter.php +++ b/scraper/filter.php @@ -178,7 +178,8 @@ function normalizeFieldList( $fieldString ) { || $word == 'QA' /* Whiteboard */ || $word == 'Restrict' /* Comments */ || $word == 'Due' /* Date */ - || $word == 'Fission' /* Milestone */) + || $word == 'Fission' /* Milestone */ + || $word == 'Regressed' /* by */) { if ($i + 1 >= count( $words )) { fail( 'Unrecognized field list (4): ' . print_r( $words, true ) ); From 8b4a9db1b88ac9e3747e93f9d4234cee3fa9d643 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 8 Apr 2019 14:47:31 +0000 Subject: [PATCH 115/141] Regressed by needs linking too --- www/dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index f810065..b2abced 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -123,7 +123,7 @@ function linkify_gh( $text, $base_repo ) { function buglinkify( $field, $text ) { global $_BASE_URL; - if ($field === 'Depends on' || $field === 'Blocks' || $field === 'Regressions') { + if ($field === 'Depends on' || $field === 'Blocks' || $field === 'Regressions' || $field === 'Regressed by') { $text = preg_replace( '/(\d+)/e', 'buglink(\'\', \'\\1\')', $text ); } return $text; From 0d452f31b5035eabb272c07576737fb7752fbbc0 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 17 May 2019 14:10:19 +0000 Subject: [PATCH 116/141] Add webcompat priority field --- scraper/filter.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scraper/filter.php b/scraper/filter.php index b496b19..c2de789 100755 --- a/scraper/filter.php +++ b/scraper/filter.php @@ -178,8 +178,9 @@ function normalizeFieldList( $fieldString ) { || $word == 'QA' /* Whiteboard */ || $word == 'Restrict' /* Comments */ || $word == 'Due' /* Date */ - || $word == 'Fission' /* Milestone */ - || $word == 'Regressed' /* by */) + || $word == 'Fission' /* Milestone */ + || $word == 'Regressed' /* by */ + || $word == 'Webcompat' /* Priority */) { if ($i + 1 >= count( $words )) { fail( 'Unrecognized field list (4): ' . print_r( $words, true ) ); From 62db13d1fabfb32d5027231aab994e62a8bd0547 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sun, 9 Jun 2019 19:57:40 +0000 Subject: [PATCH 117/141] Update field mappings --- scraper/filter.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scraper/filter.php b/scraper/filter.php index c2de789..7fac33d 100755 --- a/scraper/filter.php +++ b/scraper/filter.php @@ -222,6 +222,8 @@ function normalizeFieldList( $fieldString ) { $word = 'Whiteboard'; } else if ($word == 'QAContact') { $word = 'QA Contact'; + } else if ($word == 'Last Resolved') { + $word = 'Closed'; } $fields[] = $word; } @@ -257,7 +259,7 @@ function parseChangeTable( $fields, $rows ) { $fields[ $ixField ] = preg_replace( '/(Attachment #\d+).*/', '\1', $col1 ) . substr( $fields[ $ixField ], strlen( 'Attachment' )); $matchedStart = true; } - } else if (stripos( $fields[ $ixField ], $col1 ) === 0) { + } else if (stripos( $fields[ $ixField ], $col1 ) === 0) { // simple case match the start of the field against the column $matchedStart = true; } From fff9d634a66d18aeb8c13094f97ef30f77594538 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 15 Aug 2019 15:48:22 +0000 Subject: [PATCH 118/141] Update deprecated ssl syntax --- nginx/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index e1d4dd7..5b8f9e3 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -6,8 +6,8 @@ server { } server { - listen [::]:443; - listen 443; + listen [::]:443 ssl; + listen 443 ssl; server_name bugmash.staktrace.com; root /ebs/bugmash/www; index index.php index.html index.htm; From fa18d0e4569eff5eacb7d40a8400576bf8cd358d Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sun, 26 Jan 2020 20:55:17 +0000 Subject: [PATCH 119/141] Update use of deprecated code --- scraper/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index dd9e25b..21e907f 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -106,7 +106,7 @@ fn split_footer(msg: String) -> (String, Option) { fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { let mut title = mail.headers.get_first_value("Subject").map_err(|e| format!("{:?}", e))?.unwrap_or("(no subject)".to_string()); - title = title.trim_left_matches("Re: ").to_string(); + title = title.trim_start_matches("Re: ").to_string(); if title.starts_with("[") { if let Some(close_bracket) = title.find("]") { title = title[close_bracket + 1..].trim().to_string(); From 216648fd63364766f0dcc06605db969eef5f5aea Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 6 Feb 2020 02:41:34 +0000 Subject: [PATCH 120/141] Make scraping bugmail work, except for request type emails --- scraper/Cargo.lock | 21 ++++ scraper/Cargo.toml | 2 + scraper/src/main.rs | 263 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 282 insertions(+), 4 deletions(-) diff --git a/scraper/Cargo.lock b/scraper/Cargo.lock index 8d700ae..f90f587 100644 --- a/scraper/Cargo.lock +++ b/scraper/Cargo.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. [[package]] name = "aho-corasick" version = "0.6.8" @@ -243,6 +245,11 @@ dependencies = [ "quoted_printable 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "matches" version = "0.1.8" @@ -435,6 +442,8 @@ version = "0.1.0" dependencies = [ "mailparse 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "mysql 14.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "victoria-dom 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -561,6 +570,16 @@ name = "version_check" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "victoria-dom" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "void" version = "1.0.2" @@ -630,6 +649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7" "checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" "checksum mailparse 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d1d5ea71343df3c508b245824ec17f5c6f86f79f99ce999bf6ac51edf0465685" +"checksum maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a3b4142ab8738a78c51896f704f83c11df047ff1bda9a92a661aa6361552d93d" "checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" @@ -667,6 +687,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd70f467df6810094968e2fce0ee1bd0e87157aceb026a8c083bcf5e25b9efe4" "checksum uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e1436e58182935dcd9ce0add9ea0b558e8a87befe01c1a301e6020aeb0876363" "checksum version_check 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7716c242968ee87e5542f8021178248f267f295a5c4803beae8b8b7fd9bc6051" +"checksum victoria-dom 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "27ef2e353e03efe070c1316f2d04fba6804a40edb694951d6b7fbcbe11283a02" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" diff --git a/scraper/Cargo.toml b/scraper/Cargo.toml index 7dd5c87..9eb2e86 100644 --- a/scraper/Cargo.toml +++ b/scraper/Cargo.toml @@ -6,3 +6,5 @@ authors = ["Kartikaya Gupta "] [dependencies] mailparse = "0.6.2" mysql = "14.1.0" +victoria-dom = "0.1.2" +regex = "1" diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 21e907f..1dda3ae 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -1,6 +1,8 @@ extern crate mailparse; #[macro_use] extern crate mysql; +extern crate regex; +extern crate victoria_dom; use std::collections::hash_map::DefaultHasher; use std::env; @@ -13,6 +15,10 @@ use std::time::{SystemTime, UNIX_EPOCH}; use mailparse::{dateparse, MailHeaderMap, MailParseError, ParsedMail}; +use regex::Regex; + +use victoria_dom::DOM; + fn save_file(data: &[u8]) -> String { let time = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); let mut hasher = DefaultHasher::new(); @@ -48,18 +54,26 @@ fn get_db() -> Result { mysql::Pool::new(connstr).map_err(|e| format!("{:?}", e)) } -fn get_plain_body(mail: &ParsedMail) -> Result, MailParseError> { - if mail.ctype.mimetype == "text/plain" { +fn get_body_with_type(mail: &ParsedMail, body_type: &'static str) -> Result, MailParseError> { + if mail.ctype.mimetype == body_type { return mail.get_body().map(Some); } for subpart in &mail.subparts { - if let Some(x) = get_plain_body(subpart)? { + if let Some(x) = get_body_with_type(subpart, body_type)? { return Ok(Some(x)); } } Ok(None) } +fn get_plain_body(mail: &ParsedMail) -> Result, MailParseError> { + get_body_with_type(mail, "text/plain") +} + +fn get_html_body(mail: &ParsedMail) -> Result, MailParseError> { + get_body_with_type(mail, "text/html") +} + fn url_parts(footer: String) -> Option<(String, String, Option)> { let prefix = "https://github.com/"; let types = ["/issues/", "/pull/", "/commit/"]; @@ -161,6 +175,241 @@ fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { Ok(()) } +fn mail_header(mail: &ParsedMail, header: &str) -> Result, String> { + mail.headers.get_first_value(header).map_err(|e| format!("Unable to read mail header: {:?}", e)) +} + +fn bugmail_header(mail: &ParsedMail, header: &str) -> Result, String> { + mail_header(mail, &format!("X-Bugzilla-{}", header)) +} + +fn bugzilla_normalized_reason(mail: &ParsedMail) -> Result { + let reason = bugmail_header(mail, "Reason")?.ok_or("Unable to find Reason header")?; + let watch_reason = bugmail_header(mail, "Watch-Reason")?; + if reason.find("AssignedTo").is_some() { + Ok("AssignedTo".to_string()) + } else if reason.find("Reporter").is_some() { + Ok("Reporter".to_string()) + } else if reason.find("CC").is_some() { + Ok("CC".to_string()) + } else if reason.find("Voter").is_some() { + Ok("Voter".to_string()) + } else if reason.find("None").is_some() { + if watch_reason.is_some() { + Ok("Watch".to_string()) + } else { + Err(format!("Empty watch reason with reason {}", reason)) + } + } else { + Err(format!("Unrecognized reason {}", reason)) + } +} + +fn insert_changes(db: &mysql::Pool, id: &str, stamp: i64, reason: &str, changes: &[(String, String, String)]) -> Result<(), String> { + let mut stmt = db.prepare(r#"INSERT INTO changes (bug, stamp, reason, field, oldval, newval) + VALUES (:id, FROM_UNIXTIME(:stamp), :reason, :field, :oldval, :newval)"#).map_err(|e| format!("{:?}", e))?; + for (field, oldval, newval) in changes { + stmt.execute(params!{ + id, + stamp, + reason, + field, + oldval, + newval + }).map_err(|e| format!("{:?}", e))?; + } + Ok(()) +} + +fn insert_comments(db: &mysql::Pool, id: &str, stamp: i64, reason: &str, comments: &[(i32, String, String)]) -> Result<(), String> { + let mut stmt = db.prepare(r#"INSERT INTO comments (bug, stamp, reason, commentnum, author, comment) + VALUES (:id, FROM_UNIXTIME(:stamp), :reason, :commentnum, :author, :comment)"#).map_err(|e| format!("{:?}", e))?; + for (commentnum, author, comment) in comments { + stmt.execute(params!{ + id, + stamp, + reason, + commentnum, + author, + comment + }).map_err(|e| format!("{:?}", e))?; + } + Ok(()) +} + +fn bugmail_body(mail: &ParsedMail) -> Result { + let body = match get_html_body(mail).map_err(|e| format!("{:?}", e))? { + Some(x) => x, + None => return Err("No html body found".to_string()), + }; + Ok(DOM::new(&body)) +} + +fn bugmail_body_text(mail: &ParsedMail) -> Result { + get_plain_body(mail) + .map_err(|e| format!("{:?}", e))? + .map(|t| t.replace("\r", "")) + .ok_or("No plaintext body found".to_string()) +} + +fn scrape_bugmail_request(_mail: &ParsedMail) -> Result<(), String> { + // TODO + Err("Got a request type bugmail, not implemented yet".to_string()) +} + +fn scrape_bugmail_newbug(mail: &ParsedMail, db: &mysql::Pool, id: &str, stamp: i64) -> Result<(), String> { + let reason = bugzilla_normalized_reason(mail)?; + let dom = bugmail_body(mail)?; + let mut title = None; + let author = bugmail_header(mail, "Who")?.ok_or("Couldn't find new bug author".to_string())?; + + let body = bugmail_body_text(mail)?; + let body_re = Regex::new(r"(?s)Bug ID: .*?\n\n(.*\n\n)?-- \n").unwrap(); + let body_m = body_re.captures(&body).ok_or("Unable to capture new bug description".to_string())?; + let description = body_m.get(1).map_or("", |m| m.as_str().trim()); + + let mut tr_opt = dom.at("div.new table tr"); + while let Some(tr) = tr_opt { + let field = tr.at("td.c1") + .map(|d| d.text_all()) + .ok_or("Couldn't find new bug details".to_string())?; + if field.contains("Summary") { + title = tr.at("td.c2").map(|d| d.text_all()); + break; + } + tr_opt = tr.next(); + } + let title = title.ok_or("Couldn't find bug summary in new bug details table".to_string())?; + + db.prep_exec(r#"INSERT INTO newbugs (bug, stamp, reason, title, author, description) + VALUES (:id, FROM_UNIXTIME(:stamp), :reason, :title, :author, :description)"#, params! { + id, + stamp, + "reason" => &reason, + title, + author, + description + }).map_err(|e| format!("{:?}", e))?; + + let changes = scrape_change_table(&dom, "")?; + insert_changes(db, id, stamp, &reason, &changes)?; + + Ok(()) +} + +fn scrape_change_table(dom: &DOM, field_prefix: &str) -> Result, String> { + let mut changes = Vec::new(); + let mut change_row = dom.at("div.diffs tr.head").and_then(|d| d.next()); + while let Some(tr) = change_row { + let field = tr.at("td.c1") + .map(|d| d.text_all()) + .ok_or("Couldn't find bug change table field".to_string())?; + let old = tr.at("td.c2") + .map(|d| d.text_all()) + .ok_or("Couldn't find bug change table old value".to_string())?; + let new = tr.at("td.c2") + .and_then(|d| d.next()) + .map(|d| d.text_all()) + .ok_or("Couldn't find bug change table new value".to_string())?; + changes.push((format!("{}{}", field_prefix, field), old, new)); + change_row = tr.next(); + } + Ok(changes) +} + +fn scrape_bugmail_depchange(mail: &ParsedMail, db: &mysql::Pool, id: &str, stamp: i64) -> Result<(), String> { + let reason = bugzilla_normalized_reason(mail)?; + let dom = bugmail_body(mail)?; + let depbug_text = dom.at("body > b").map(|d| d.text_all()).ok_or("Unable to find bug dependency sentence")?; + let depbug_re = Regex::new(r"(?i)bug (\d+) depends on bug(?: | )(\d+), which changed state.").unwrap(); + let depbug_m = depbug_re.captures(&depbug_text).ok_or("Dependency sentence didn't match expected regex".to_string())?; + if &depbug_m[1] != id { + return Err("Dependency sentence referred to wrong bug id".to_string()); + } + + let depbug = &depbug_m[2]; + let changes = scrape_change_table(&dom, &format!("depbug-{}-", depbug))?; + insert_changes(db, id, stamp, &reason, &changes) +} + +fn scrape_comments(mail: &ParsedMail) -> Result, String> { + let mut comment_tuples = Vec::new(); + let body = bugmail_body_text(mail)?; + let body_re = Regex::new(r"(?sU)-- Comment #(\d+) from ([^\n]*) ---\n(.*)\n\n--").unwrap(); + for capture in body_re.captures_iter(&body) { + let comment_num = &capture[1]; + let mut author = String::from(&capture[2]); + let comment_text = &capture[3]; + let stripped_len = author.len() - " YYYY-mm-dd HH:ii:ss ZZZ".len(); + author.truncate(stripped_len); + if let Some(idx) = author.find('<') { + author.truncate(idx); + } + comment_tuples.push((comment_num.parse::().map_err(|e| format!("{:?}", e))?, author, comment_text.to_string())); + } + Ok(comment_tuples) +} + +fn scrape_bugmail_change(mail: &ParsedMail, db: &mysql::Pool, id: &str, stamp: i64) -> Result<(), String> { + let reason = bugzilla_normalized_reason(mail)?; + let dom = bugmail_body(mail)?; + + let changes = scrape_change_table(&dom, "")?; + insert_changes(db, id, stamp, &reason, &changes)?; + + let comments = scrape_comments(mail)?; + insert_comments(db, id, stamp, &reason, &comments)?; + + if changes.len() == 0 && comments.len() == 0 { + return Err("Unable to extract meaningful data from changed email".to_string()); + } + Ok(()) +} + +fn scrape_bugzilla_mail(bz_type: &str, mail: &ParsedMail) -> Result<(), String> { + if bz_type == "nag" { + return Ok(()); + } + + let secure = bugmail_header(mail, "Secure-Email")?.is_some(); + let id = bugmail_header(mail, "ID")?.ok_or("Unable to find bug id".to_string())?; + let stamp = mail_header(mail, "Date")?.map(|v| dateparse(&v).unwrap_or(0)).unwrap_or(0); + + let subject = mail_header(mail, "Subject")?.ok_or("Unable to find subject header".to_string())?; + let subject_re = Regex::new(r"(?sU)\[Bug \d+\] (.*)( : \[Attachment.*)?$").unwrap(); + let subject_m = subject_re.captures(&subject).ok_or("Subject header didn't match expected regex".to_string())?; + let title = &subject_m[1]; + + let db = get_db()?; + db.prep_exec(r#"INSERT INTO metadata (bug, stamp, title, secure, note) + VALUES (:id, FROM_UNIXTIME(:stamp), :title, :secure, "") + ON DUPLICATE KEY UPDATE stamp=VALUES(stamp), title=VALUES(title), secure=VALUES(secure)"#, params! { + "id" => &id, + stamp, + title, + secure, + }).map_err(|e| format!("{:?}", e))?; + + if secure { + // you haven't set a PGP/GPG key and this is for a secure bug, so there's no data in it. + let reason = bugzilla_normalized_reason(mail)?; + let unknown = "Unknown"; + let changes = vec![ (unknown.to_string(), unknown.to_string(), unknown.to_string()) ]; + insert_changes(&db, &id, stamp, &reason, &changes)?; + Ok(()) + } else if bz_type == "request" { + scrape_bugmail_request(mail) + } else if bz_type == "new" { + scrape_bugmail_newbug(mail, &db, &id, stamp) + } else if bz_type == "dep_changed" { + scrape_bugmail_depchange(mail, &db, &id, stamp) + } else if bz_type == "changed" { + scrape_bugmail_change(mail, &db, &id, stamp) + } else { + Err(format!("Unknown bugmail type {}", bz_type)) + } +} + fn main() { let mut input = Vec::new(); { @@ -175,7 +424,13 @@ fn main() { let github_reason = mail.headers.get_first_value("X-GitHub-Reason").unwrap_or_else(|e| fail(&saved_file, "Unable to read mail header", format!("{:?}", e))); if github_reason.is_some() { - scrape_github_mail(&mail).unwrap_or_else(|e| fail(&saved_file, "Error while scraping mail", e)); + scrape_github_mail(&mail).unwrap_or_else(|e| fail(&saved_file, "Error while scraping github mail", e)); + fs::remove_file(&saved_file).unwrap_or_else(|e| fail(&saved_file, "Error removing file after processing", format!("{:?}", e))); + } + + let bugzilla_type = mail.headers.get_first_value("X-Bugzilla-Type").unwrap_or_else(|e| fail(&saved_file, "Unable to read mail header", format!("{:?}", e))); + if let Some(bz_type) = bugzilla_type { + scrape_bugzilla_mail(&bz_type, &mail).unwrap_or_else(|e| fail(&saved_file, "Error while scraping bugzilla mail", e)); fs::remove_file(&saved_file).unwrap_or_else(|e| fail(&saved_file, "Error removing file after processing", format!("{:?}", e))); } } From 776501b52e087b80be60471ed06e5c8af7e0eb5e Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sat, 15 Feb 2020 00:15:30 +0000 Subject: [PATCH 121/141] Add support for needinfo request emails --- scraper/src/main.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 1dda3ae..fa16b11 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -252,9 +252,23 @@ fn bugmail_body_text(mail: &ParsedMail) -> Result { .ok_or("No plaintext body found".to_string()) } -fn scrape_bugmail_request(_mail: &ParsedMail) -> Result<(), String> { - // TODO - Err("Got a request type bugmail, not implemented yet".to_string()) +fn scrape_bugmail_request(mail: &ParsedMail, db: &mysql::Pool, id: &str, stamp: i64) -> Result<(), String> { + let requestee = bugmail_header(mail, "Flag-Requestee")?; + if requestee.is_none() { + return Err("Got a request type bugmail with no requestee, not implemented yet".to_string()); + } + let flag = mail_header(mail, "Subject")? + .and_then(|s| s.split_ascii_whitespace().next().map(str::to_string)) + .ok_or("Got a request type bugmail with no subject, not implemented yet".to_string())?; + db.prep_exec(r#"INSERT INTO requests (bug, stamp, attachment, title, flag) + VALUES (:id, FROM_UNIXTIME(:stamp), :attachment, :title, :flag)"#, params! { + id, + stamp, + "attachment" => 0, + "title" => "", + flag + }).map_err(|e| format!("{:?}", e))?; + Ok(()) } fn scrape_bugmail_newbug(mail: &ParsedMail, db: &mysql::Pool, id: &str, stamp: i64) -> Result<(), String> { @@ -398,7 +412,7 @@ fn scrape_bugzilla_mail(bz_type: &str, mail: &ParsedMail) -> Result<(), String> insert_changes(&db, &id, stamp, &reason, &changes)?; Ok(()) } else if bz_type == "request" { - scrape_bugmail_request(mail) + scrape_bugmail_request(mail, &db, &id, stamp) } else if bz_type == "new" { scrape_bugmail_newbug(mail, &db, &id, stamp) } else if bz_type == "dep_changed" { From c622093a80357de130d468a863a2f05e8a475074 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sat, 15 Feb 2020 00:15:45 +0000 Subject: [PATCH 122/141] Look in a different dir for error emails --- www/dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index b2abced..52f3214 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -369,7 +369,7 @@ function safeGet( &$array, $index ) { $_DB->close(); $errors = 0; -$files = scandir( $BUGMASH_DIR ); +$files = scandir( $_SERVER['DOCUMENT_ROOT'] . "/../../" ); foreach ($files AS $file) { if (strpos( strrev( $file ), "rre." ) === 0) { $errors++; From 78a25c1c6f597072751cbfc62cb4a295c3c85bb6 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 21 Feb 2020 15:29:56 +0000 Subject: [PATCH 123/141] Add handling for needinfo cancellation --- scraper/src/main.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index fa16b11..dcd4503 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -254,12 +254,24 @@ fn bugmail_body_text(mail: &ParsedMail) -> Result { fn scrape_bugmail_request(mail: &ParsedMail, db: &mysql::Pool, id: &str, stamp: i64) -> Result<(), String> { let requestee = bugmail_header(mail, "Flag-Requestee")?; - if requestee.is_none() { - return Err("Got a request type bugmail with no requestee, not implemented yet".to_string()); - } let flag = mail_header(mail, "Subject")? .and_then(|s| s.split_ascii_whitespace().next().map(str::to_string)) .ok_or("Got a request type bugmail with no subject, not implemented yet".to_string())?; + if requestee.is_none() { + let cancelled = mail_header(mail, "Subject")? + .and_then(|s| s.split_ascii_whitespace().nth(1).map(str::to_string)) + .map(|s| s.starts_with("canceled")) + .ok_or("Got a request type bugmail with no requestee and unexpected subject".to_string())?; + if cancelled { + db.prep_exec(r#"UPDATE requests SET cancelled=1 WHERE bug=:id and attachment=:attachment AND flag=:flag"#, params! { + id, + "attachment" => 0, + flag + }).map_err(|e| format!("{:?}", e))?; + return Ok(()); + } + return Err("Got a request type bugmail with no requestee, not implemented yet".to_string()); + } db.prep_exec(r#"INSERT INTO requests (bug, stamp, attachment, title, flag) VALUES (:id, FROM_UNIXTIME(:stamp), :attachment, :title, :flag)"#, params! { id, From ba88843d4428a4b8c6bd5664d81c4921936bf3ad Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 27 Feb 2020 17:16:12 +0000 Subject: [PATCH 124/141] Secure request emails don't have a reason --- scraper/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index dcd4503..ece8b25 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -416,15 +416,15 @@ fn scrape_bugzilla_mail(bz_type: &str, mail: &ParsedMail) -> Result<(), String> secure, }).map_err(|e| format!("{:?}", e))?; - if secure { + if bz_type == "request" { + scrape_bugmail_request(mail, &db, &id, stamp) + } else if secure { // you haven't set a PGP/GPG key and this is for a secure bug, so there's no data in it. let reason = bugzilla_normalized_reason(mail)?; let unknown = "Unknown"; let changes = vec![ (unknown.to_string(), unknown.to_string(), unknown.to_string()) ]; insert_changes(&db, &id, stamp, &reason, &changes)?; Ok(()) - } else if bz_type == "request" { - scrape_bugmail_request(mail, &db, &id, stamp) } else if bz_type == "new" { scrape_bugmail_newbug(mail, &db, &id, stamp) } else if bz_type == "dep_changed" { From 8139f0105a69c9309fe3932d0ae88be6becff9cf Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sun, 22 Mar 2020 16:51:44 +0000 Subject: [PATCH 125/141] Add support for granted requests --- scraper/src/main.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index ece8b25..87ecff4 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -258,11 +258,10 @@ fn scrape_bugmail_request(mail: &ParsedMail, db: &mysql::Pool, id: &str, stamp: .and_then(|s| s.split_ascii_whitespace().next().map(str::to_string)) .ok_or("Got a request type bugmail with no subject, not implemented yet".to_string())?; if requestee.is_none() { - let cancelled = mail_header(mail, "Subject")? + let action = mail_header(mail, "Subject")? .and_then(|s| s.split_ascii_whitespace().nth(1).map(str::to_string)) - .map(|s| s.starts_with("canceled")) .ok_or("Got a request type bugmail with no requestee and unexpected subject".to_string())?; - if cancelled { + if action.starts_with("canceled") { db.prep_exec(r#"UPDATE requests SET cancelled=1 WHERE bug=:id and attachment=:attachment AND flag=:flag"#, params! { id, "attachment" => 0, @@ -270,6 +269,22 @@ fn scrape_bugmail_request(mail: &ParsedMail, db: &mysql::Pool, id: &str, stamp: }).map_err(|e| format!("{:?}", e))?; return Ok(()); } + if action.starts_with("granted:") { + let author_email = bugmail_header(mail, "Who")?; + db.prep_exec(r#"INSERT INTO reviews (bug, stamp, attachment, title, flag, author, authoremail, granted, comment) + VALUES (:id, FROM_UNIXTIME(:stamp), :attachment, :title, :flag, :author, :author_email, :granted, :comment)"#, params! { + id, + stamp, + "attachment" => 0, + "title" => "", + flag, + "author" => "", + author_email, + "granted" => 1, + "comment" => "", + }).map_err(|e| format!("{:?}", e))?; + return Ok(()); + } return Err("Got a request type bugmail with no requestee, not implemented yet".to_string()); } db.prep_exec(r#"INSERT INTO requests (bug, stamp, attachment, title, flag) From 0b7fbd6ff764d6d598de521f4f8e631382fd6724 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 6 Apr 2020 18:39:27 +0000 Subject: [PATCH 126/141] Add support for denied requests --- scraper/src/main.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 87ecff4..42bedf8 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -285,6 +285,22 @@ fn scrape_bugmail_request(mail: &ParsedMail, db: &mysql::Pool, id: &str, stamp: }).map_err(|e| format!("{:?}", e))?; return Ok(()); } + if action == "not" { + let author_email = bugmail_header(mail, "Who")?; + db.prep_exec(r#"INSERT INTO reviews (bug, stamp, attachment, title, flag, author, authoremail, granted, comment) + VALUES (:id, FROM_UNIXTIME(:stamp), :attachment, :title, :flag, :author, :author_email, :granted, :comment)"#, params! { + id, + stamp, + "attachment" => 0, + "title" => "", + flag, + "author" => "", + author_email, + "granted" => 0, + "comment" => "", + }).map_err(|e| format!("{:?}", e))?; + return Ok(()); + } return Err("Got a request type bugmail with no requestee, not implemented yet".to_string()); } db.prep_exec(r#"INSERT INTO requests (bug, stamp, attachment, title, flag) From 53a451725d4178ac70d591f51785e54192c6fbe1 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 16 Oct 2020 15:46:59 +0000 Subject: [PATCH 127/141] Add support for failed CI runs --- scraper/src/main.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 42bedf8..1df698d 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -108,16 +108,27 @@ fn url_parts(footer: String) -> Option<(String, String, Option)> { hash)); } -fn split_footer(msg: String) -> (String, Option) { +fn split_footer(msg: &str) -> (String, Option) { // Avoid trying to match newlines directly since they can be either \r\n or \n let ix = msg.rfind("You are receiving this because") .and_then(|ix| msg[0..ix].rfind("-- ")); match ix { Some(ix) => (String::from(&msg[0..ix]), Some(String::from(&msg[ix..]))), - None => (msg, None), + None => (String::from(msg), None), } } +fn first_github_url(body: &str) -> Option<(String, String, Option)> { + let prefix = "https://github.com/"; + let ix = body.find(prefix)? + prefix.len(); + let end_ix = ix + body[ix..].find(char::is_whitespace)?; + let urlpath = &body[ix..end_ix]; + let slash = "/"; + let org_end_ix = urlpath.find(slash)?; + let repo_end_ix = org_end_ix + urlpath[org_end_ix..].find(slash)?; + Some((String::from(&urlpath[0..repo_end_ix]), String::from(&urlpath[repo_end_ix + 1..]), None)) +} + fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { let mut title = mail.headers.get_first_value("Subject").map_err(|e| format!("{:?}", e))?.unwrap_or("(no subject)".to_string()); title = title.trim_start_matches("Re: ").to_string(); @@ -135,17 +146,21 @@ fn scrape_github_mail(mail: &ParsedMail) -> Result<(), String> { let sender = mail.headers.get_first_value("X-GitHub-Sender").map_err(|e| format!("{:?}", e))?.unwrap_or("Unknown".to_string()); let reason = match mail.headers.get_first_value("X-GitHub-Reason").map_err(|e| format!("{:?}", e))? { Some(ref s) if s == "review_requested" => "review", + Some(ref s) if s == "ci_activity" => "review", Some(ref s) if s == "author" => "Reporter", Some(ref s) if s == "mention" => "CC", _ => "Watch", }; let stamp = mail.headers.get_first_value("Date").map_err(|e| format!("{:?}", e))?.map(|v| dateparse(&v).unwrap_or(0)).unwrap_or(0); - let (comment, footer) = match get_plain_body(mail).map_err(|e| format!("{:?}", e))? { - Some(x) => split_footer(x), + let plain_body = match get_plain_body(mail).map_err(|e| format!("{:?}", e))? { + Some(x) => x, None => return Err("No plaintext body found".to_string()), }; + let (comment, footer) = split_footer(&plain_body); let footer = footer.ok_or("Unable to find footer".to_string())?; - let (repo, issue, hash) = url_parts(footer).ok_or("Unable to extract URL parts".to_string())?; + let (repo, issue, hash) = url_parts(footer) + .or_else(|| first_github_url(&plain_body)) + .ok_or("Unable to extract URL parts".to_string())?; let hash = hash.unwrap_or(String::from("")); let id = format!("{}#{}", repo, issue); From eee90055e0b0c5ad18e83cb6860e5970990af497 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 23 Oct 2020 20:39:45 +0000 Subject: [PATCH 128/141] Set up initial phabricator scraping --- scraper/src/main.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 1df698d..501c335 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -482,6 +482,10 @@ fn scrape_bugzilla_mail(bz_type: &str, mail: &ParsedMail) -> Result<(), String> } } +fn scrape_phabricator_mail(mail: &ParsedMail) -> Result<(), String> { + Err(format!("Unhandled phabmail")) +} + fn main() { let mut input = Vec::new(); { @@ -505,4 +509,10 @@ fn main() { scrape_bugzilla_mail(&bz_type, &mail).unwrap_or_else(|e| fail(&saved_file, "Error while scraping bugzilla mail", e)); fs::remove_file(&saved_file).unwrap_or_else(|e| fail(&saved_file, "Error removing file after processing", format!("{:?}", e))); } + + let phabricator = mail.headers.get_first_value("X-Phabricator-Sent-This-Message").unwrap_or_else(|e| fail(&saved_file, "Unable to read mail header", format!("{:?}", e))); + if phabricator.is_some() { + scrape_phabricator_mail(&mail).unwrap_or_else(|e| fail(&saved_file, "Error while scraping phab mail", e)); + fs::remove_file(&saved_file).unwrap_or_else(|e| fail(&saved_file, "Error removing file after processing", format!("{:?}", e))); + } } From 5e5b92329a6511a4cae27b9ee241d92d3197684c Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 29 Oct 2020 08:53:26 +0000 Subject: [PATCH 129/141] Add table for phab diffs --- schemas/tables.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/schemas/tables.sql b/schemas/tables.sql index 7959a4b..e20ed68 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -92,6 +92,20 @@ CREATE TABLE `gh_issues` ( KEY (`viewed`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +CREATE TABLE `phab_diffs` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `revision` varchar(32) NOT NULL, + `stamp` datetime NOT NULL, + `viewed` tinyint(1) NOT NULL DEFAULT 0, + `reason` varchar(10) NOT NULL, + `author` varchar(255) NOT NULL, + `comment` mediumtext NOT NULL, + PRIMARY KEY (`id`), + KEY (`revision`), + KEY (`stamp`), + KEY (`viewed`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + CREATE TABLE `metadata` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `bug` varchar(128) NOT NULL, From ab79163c2df2f49eae35aa4571d77a660db5a392 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 29 Oct 2020 09:01:07 +0000 Subject: [PATCH 130/141] Add support for phab_diff table to dashboard --- www/common.php | 14 +++++++-- www/common.php.bk | 73 +++++++++++++++++++++++++++++++++++++++++++++++ www/dashboard.php | 22 +++++++++++++- www/wipe.php | 3 ++ 4 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 www/common.php.bk diff --git a/www/common.php b/www/common.php index d9940ff..58fc5b4 100644 --- a/www/common.php +++ b/www/common.php @@ -22,6 +22,7 @@ function fail( $message ) { $BUGMASH_DIR = $_SERVER['DOCUMENT_ROOT'] . '/../scraper'; include_once( $BUGMASH_DIR . '/config.php' ); $_GH_BASE_URL = "https://github.com/"; +$_PHAB_BASE_URL = "https://phabricator.services.mozilla.com"; function updateTags( $newTags ) { global $_DB; @@ -64,18 +65,25 @@ function isGithubCommit( $bugid ) { return (strlen( $bugid ) - strpos( $bugid, '#' )) >= 40; } +function isPhabDiff( $bugid ) { + return ($bugid[0] == 'D'); +} + function makeBugLink( $bugid ) { - global $_BASE_URL, $_GH_BASE_URL; + global $_BASE_URL, $_GH_BASE_URL, $_PHAB_BASE_URL; if (isGithubIssue( $bugid )) { $type = isGithubCommit( $bugid ) ? '/commit/' : '/issues/'; return sprintf( '%s', $_GH_BASE_URL . str_replace( '#', $type, $bugid ), - $bugid); + $bugid ); + } else if (isPhabDiff( $bugid )) { + return sprintf( '%s', + $_PHAB_BASE_URL, $bugid, $bugid ); } else { return sprintf( '%s', $_BASE_URL . '/show_bug.cgi?id=' . $bugid, - "Bug " . $bugid); + "Bug " . $bugid ); } } diff --git a/www/common.php.bk b/www/common.php.bk new file mode 100644 index 0000000..c19551c --- /dev/null +++ b/www/common.php.bk @@ -0,0 +1,73 @@ + $value) { + $_POST[$key] = stripslashes($value); + } + foreach ($_GET AS $key => $value) { + $_GET[$key] = stripslashes($value); + } + foreach ($_COOKIE AS $key => $value) { + $_COOKIE[$key] = stripslashes($value); + } +} + +function fail( $message ) { + error_log( $message ); + header( 'HTTP/500 Error!' ); + print $message; + exit( 0 ); +} + +$BUGMASH_DIR = $_SERVER['DOCUMENT_ROOT'] . '/../scraper'; +include_once( $BUGMASH_DIR . '/config.php' ); +$_GH_BASE_URL = "https://github.com/"; + +function updateTags( $newTags ) { + global $_DB; + + $stmt = $_DB->prepare( 'DELETE FROM tags WHERE bug=?' ); + if ($_DB->errno) fail( 'Error preparing tag deletion: ' . $_DB->error ); + foreach ($newTags AS $bug => $tagList) { + $stmt->bind_param( 's', $bug ); + $stmt->execute(); + if ($stmt->errno) fail( 'Error inserting to metadata: ' . $stmt->error ); + } + $stmt->close(); + + $stmt = $_DB->prepare( 'INSERT INTO tags (bug, tag) VALUES (?, ?)' ); + if ($_DB->errno) fail( 'Error preparing tag insertion: ' . $_DB->error ); + foreach ($newTags AS $bug => $tagList) { + foreach (explode( ',', $tagList ) AS $tag) { + $tag = trim( $tag ); + if (strlen( $tag ) > 0) { + $stmt->bind_param( 'ss', $bug, $tag ); + $stmt->execute(); + if ($stmt->errno) fail( 'Error inserting to metadata: ' . $stmt->error ); + } + } + } + $stmt->close(); +} + +function escapeHTML( $stuff ) { + $stuff = str_replace( '&', '&', $stuff ); + $stuff = str_replace( array( '<', '>', '"' ), array( '<', '>', '"' ), $stuff ); + return $stuff; +} + +//function bugLink( $bugid ) { +// global $_BASE_URL, $_GH_BASE_URL; +// +// if (strpos( $bugid, '#' ) !== FALSE) { +// return sprintf( '%s', +// $_GH_BASE_URL . str_replace( '#', '/issues/', $bugid ), +// $bugid); +// } else { +// return sprintf( '%s', +// $_BASE_URL . '/show_bug.cgi?id=' . $bugid, +// "Bug " . $bugid); +// } +//} + +?> diff --git a/www/dashboard.php b/www/dashboard.php index 52f3214..8cce777 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -121,6 +121,11 @@ function linkify_gh( $text, $base_repo ) { return $text; } +function linkify_phab( $text ) { + $text = preg_replace( '#\[(.*?)\]\((https?://.*?)\)#i', '$1', $text ); // markdown links + return $text; +} + function buglinkify( $field, $text ) { global $_BASE_URL; if ($field === 'Depends on' || $field === 'Blocks' || $field === 'Regressions' || $field === 'Regressed by') { @@ -335,10 +340,25 @@ function safeGet( &$array, $index ) { $reasons[ $bugid ][] = $row['reason']; } +$result = loadTable( 'phab_diffs' ); +while ($row = $result->fetch_assoc()) { + $numRows++; + $stamp = strtotime( $row['stamp'] ); + $bugid = $row['revision']; + initEmpty( $bblocks, $bugid, $stamp ); + $bblocks[ $bugid ][ $stamp ] .= sprintf( '
%s said:
%s
', + $row['id'], + escapeHTML( $row['author'] ), + $_PHAB_BASE_URL, + $row['revision'], + linkify_phab( escapeHTML( $row['comment'] ) ) ) . "\n"; + $reasons[ $bugid ][] = $row['reason']; +} + foreach ($bblocks AS $bug => &$block) { ksort( $block, SORT_NUMERIC ); $touchTime = key( $block ); - $identifier = (isGithubIssue( $bug ) ? 'gh_' : 'bug') . $bug; + $identifier = (isGithubIssue( $bug ) ? 'gh_' : (isPhabDiff( $bug ) ? 'pd_' : 'bug')) . $bug; $block = sprintf( '
' . 'X ' . '%s' diff --git a/www/wipe.php b/www/wipe.php index cb6d995..8c9a31f 100644 --- a/www/wipe.php +++ b/www/wipe.php @@ -32,6 +32,9 @@ case 'g': $table = 'gh_issues'; break; + case 'd': + $table = 'phab_diffs'; + break; } $rowId = intval( substr( $id, 1 ) ); $_DB->query( "UPDATE {$table} SET viewed=1 WHERE id={$rowId}" ); From bd744f732768c785d3169be04840fa319e1470ca Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Thu, 29 Oct 2020 09:02:04 +0000 Subject: [PATCH 131/141] Implement phabricator mail scraping (untested) --- scraper/src/main.rs | 48 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 501c335..138fa7e 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -483,7 +483,53 @@ fn scrape_bugzilla_mail(bz_type: &str, mail: &ParsedMail) -> Result<(), String> } fn scrape_phabricator_mail(mail: &ParsedMail) -> Result<(), String> { - Err(format!("Unhandled phabmail")) + let stamp = mail_header(mail, "Date")?.map(|v| dateparse(&v).unwrap_or(0)).unwrap_or(0); + + let stamps = mail_header(mail, "X-Phabricator-Stamps")?.ok_or("Unable to get stamps header".to_string())?; + let actor_re = Regex::new(r"actor\((.*?)\)").unwrap(); + let actor_m = actor_re.captures(&stamps).ok_or("Stamps header didn't match actor regex".to_string())?; + let actor = &actor_m[1]; + + let reason = match &stamps { + s if s.contains("reviewer(@kats)") => "review", + s if s.contains("author(@kats)") => "Reporter", + _ => "CC", + }; + + let subject = mail_header(mail, "Subject")?.ok_or("Unable to find subject header".to_string())?; + let subject_re = Regex::new(r"Differential.* (D\d+): (.*)").unwrap(); + let subject_m = subject_re.captures(&subject).ok_or("Subject header didn't match differential regex".to_string())?; + let phab = &subject_m[1]; + let title = &subject_m[2]; + + let db = get_db()?; + db.prep_exec(r#"INSERT INTO metadata (bug, stamp, title, secure, note) + VALUES (:id, FROM_UNIXTIME(:stamp), :title, :secure, "") + ON DUPLICATE KEY UPDATE stamp=VALUES(stamp), title=VALUES(title), secure=VALUES(secure)"#, params! { + "id" => &phab, + stamp, + title, + "secure" => false, + }).map_err(|e| format!("{:?}", e))?; + + let plain_body = match get_plain_body(mail).map_err(|e| format!("{:?}", e))? { + Some(x) => x, + None => return Err("No plaintext body found".to_string()), + }; + + let result = db.prep_exec(r#"INSERT INTO phab_diffs (revision, stamp, reason, author, comment) + VALUES (:revision, FROM_UNIXTIME(:stamp), :reason, :actor, :comment)"#, params! { + "revision" => &phab, + stamp, + reason, + actor, + "comment" => &plain_body, + }).map_err(|e| format!("{:?}", e))?; + if result.affected_rows() != 1 { + return Err(format!("Affected row count for phab_diffs was {}, not 1", result.affected_rows())); + } + + Ok(()) } fn main() { From 99b20a8838ea24427aaf82f5a1f5a73f48447a4b Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 30 Oct 2020 14:29:53 +0000 Subject: [PATCH 132/141] Linkify all links for phabmail --- www/dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index 8cce777..8b50ae9 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -122,7 +122,7 @@ function linkify_gh( $text, $base_repo ) { } function linkify_phab( $text ) { - $text = preg_replace( '#\[(.*?)\]\((https?://.*?)\)#i', '$1', $text ); // markdown links + $text = preg_replace( '#(https?://\S+)#i', '$1', $text ); return $text; } From 20e58bc7e2fb05c1546f2120fdba739f1f96669d Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 30 Oct 2020 14:33:23 +0000 Subject: [PATCH 133/141] Trim stuff after revision detail --- scraper/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 138fa7e..15bebc1 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -512,10 +512,13 @@ fn scrape_phabricator_mail(mail: &ParsedMail) -> Result<(), String> { "secure" => false, }).map_err(|e| format!("{:?}", e))?; - let plain_body = match get_plain_body(mail).map_err(|e| format!("{:?}", e))? { + let mut plain_body = match get_plain_body(mail).map_err(|e| format!("{:?}", e))? { Some(x) => x, None => return Err("No plaintext body found".to_string()), }; + if let Some(ix) = plain_body.find("\nREVISION DETAIL") { + plain_body = plain_body[0..ix].to_string(); + } let result = db.prep_exec(r#"INSERT INTO phab_diffs (revision, stamp, reason, author, comment) VALUES (:revision, FROM_UNIXTIME(:stamp), :reason, :actor, :comment)"#, params! { From db151921cd3f2404a145d94427cabde6f55ca8ed Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 30 Oct 2020 14:59:42 +0000 Subject: [PATCH 134/141] Use a unique letter for phab_diffs --- www/dashboard.php | 2 +- www/wipe.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/www/dashboard.php b/www/dashboard.php index 8b50ae9..450d3e3 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -346,7 +346,7 @@ function safeGet( &$array, $index ) { $stamp = strtotime( $row['stamp'] ); $bugid = $row['revision']; initEmpty( $bblocks, $bugid, $stamp ); - $bblocks[ $bugid ][ $stamp ] .= sprintf( '
%s said:
%s
', + $bblocks[ $bugid ][ $stamp ] .= sprintf( '
%s said:
%s
', $row['id'], escapeHTML( $row['author'] ), $_PHAB_BASE_URL, diff --git a/www/wipe.php b/www/wipe.php index 8c9a31f..9b539f0 100644 --- a/www/wipe.php +++ b/www/wipe.php @@ -32,7 +32,7 @@ case 'g': $table = 'gh_issues'; break; - case 'd': + case 'p': $table = 'phab_diffs'; break; } From 1564dcd9fbb027299738a80e1458b162c1fe6ffa Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Fri, 6 Nov 2020 03:24:44 +0000 Subject: [PATCH 135/141] Add support for diffusion things too --- scraper/src/main.rs | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 15bebc1..53b6e41 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -496,11 +496,27 @@ fn scrape_phabricator_mail(mail: &ParsedMail) -> Result<(), String> { _ => "CC", }; + let mut plain_body = match get_plain_body(mail).map_err(|e| format!("{:?}", e))? { + Some(x) => x, + None => return Err("No plaintext body found".to_string()), + }; + if let Some(ix) = plain_body.find("\nREVISION DETAIL") { + plain_body = plain_body[0..ix].to_string(); + } + let subject = mail_header(mail, "Subject")?.ok_or("Unable to find subject header".to_string())?; - let subject_re = Regex::new(r"Differential.* (D\d+): (.*)").unwrap(); - let subject_m = subject_re.captures(&subject).ok_or("Subject header didn't match differential regex".to_string())?; - let phab = &subject_m[1]; - let title = &subject_m[2]; + + let (phab, title) = if stamps.contains("application(Diffusion)") { + let subject_re = Regex::new(r"Diffusion[^:]*:(.*)").unwrap(); + let subject_m = subject_re.captures(&subject).ok_or("Subject header didn't match diffusion regex".to_string())?; + let diff_re = Regex::new(r"Differential Revision: https://phabricator.services.mozilla.com/(D\d+)").unwrap(); + let diff_m = diff_re.captures(&plain_body).ok_or("Diffusion body didn't match differential regex".to_string())?; + (diff_m[1].to_string(), subject_m[1].to_string()) + } else { + let subject_re = Regex::new(r"Differential.* (D\d+): (.*)").unwrap(); + let subject_m = subject_re.captures(&subject).ok_or("Subject header didn't match differential regex".to_string())?; + (subject_m[1].to_string(), subject_m[2].to_string()) + }; let db = get_db()?; db.prep_exec(r#"INSERT INTO metadata (bug, stamp, title, secure, note) @@ -508,18 +524,10 @@ fn scrape_phabricator_mail(mail: &ParsedMail) -> Result<(), String> { ON DUPLICATE KEY UPDATE stamp=VALUES(stamp), title=VALUES(title), secure=VALUES(secure)"#, params! { "id" => &phab, stamp, - title, + "title" => &title, "secure" => false, }).map_err(|e| format!("{:?}", e))?; - let mut plain_body = match get_plain_body(mail).map_err(|e| format!("{:?}", e))? { - Some(x) => x, - None => return Err("No plaintext body found".to_string()), - }; - if let Some(ix) = plain_body.find("\nREVISION DETAIL") { - plain_body = plain_body[0..ix].to_string(); - } - let result = db.prep_exec(r#"INSERT INTO phab_diffs (revision, stamp, reason, author, comment) VALUES (:revision, FROM_UNIXTIME(:stamp), :reason, :actor, :comment)"#, params! { "revision" => &phab, From 50fd7620c344521a5cf07ce9a0df38d8c38db9f9 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sun, 22 Nov 2020 01:08:26 +0000 Subject: [PATCH 136/141] Add some support for diffusion revert emails --- scraper/src/main.rs | 73 ++++++++++++++++++++++++++++++++------------- www/search.php | 3 +- 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 53b6e41..d9f8520 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -506,12 +506,22 @@ fn scrape_phabricator_mail(mail: &ParsedMail) -> Result<(), String> { let subject = mail_header(mail, "Subject")?.ok_or("Unable to find subject header".to_string())?; + let mut bugzilla = false; let (phab, title) = if stamps.contains("application(Diffusion)") { let subject_re = Regex::new(r"Diffusion[^:]*:(.*)").unwrap(); let subject_m = subject_re.captures(&subject).ok_or("Subject header didn't match diffusion regex".to_string())?; let diff_re = Regex::new(r"Differential Revision: https://phabricator.services.mozilla.com/(D\d+)").unwrap(); - let diff_m = diff_re.captures(&plain_body).ok_or("Diffusion body didn't match differential regex".to_string())?; - (diff_m[1].to_string(), subject_m[1].to_string()) + if let Some(diff_m) = diff_re.captures(&plain_body) { + (diff_m[1].to_string(), subject_m[1].to_string()) + } else { + let bug_re = Regex::new(r"(?i)bug (\d+)").unwrap(); + let bug_m = bug_re.captures(&subject_m[1]).ok_or("Unrecognized diffusion email type".to_string())?; + bugzilla = true; + if let Some(ix) = plain_body.find("\nBRANCHES") { + plain_body = plain_body[0..ix].to_string(); + } + (bug_m[1].to_string(), subject_m[1].to_string()) + } } else { let subject_re = Regex::new(r"Differential.* (D\d+): (.*)").unwrap(); let subject_m = subject_re.captures(&subject).ok_or("Subject header didn't match differential regex".to_string())?; @@ -519,25 +529,46 @@ fn scrape_phabricator_mail(mail: &ParsedMail) -> Result<(), String> { }; let db = get_db()?; - db.prep_exec(r#"INSERT INTO metadata (bug, stamp, title, secure, note) - VALUES (:id, FROM_UNIXTIME(:stamp), :title, :secure, "") - ON DUPLICATE KEY UPDATE stamp=VALUES(stamp), title=VALUES(title), secure=VALUES(secure)"#, params! { - "id" => &phab, - stamp, - "title" => &title, - "secure" => false, - }).map_err(|e| format!("{:?}", e))?; - - let result = db.prep_exec(r#"INSERT INTO phab_diffs (revision, stamp, reason, author, comment) - VALUES (:revision, FROM_UNIXTIME(:stamp), :reason, :actor, :comment)"#, params! { - "revision" => &phab, - stamp, - reason, - actor, - "comment" => &plain_body, - }).map_err(|e| format!("{:?}", e))?; - if result.affected_rows() != 1 { - return Err(format!("Affected row count for phab_diffs was {}, not 1", result.affected_rows())); + if bugzilla { + // Don't update anything on duplicate key, leave bugmail as source of truth + db.prep_exec(r#"INSERT IGNORE INTO metadata (bug, stamp, title, secure, note) + VALUES (:id, FROM_UNIXTIME(:stamp), :title, :secure, "")"#, params! { + "id" => &phab, + stamp, + "title" => &title, + "secure" => false, + }).map_err(|e| format!("{:?}", e))?; + let result = db.prep_exec(r#"INSERT INTO comments (bug, stamp, reason, commentnum, author, comment) + VALUES (:id, FROM_UNIXTIME(:stamp), :reason, 0, :actor, :comment)"#, params! { + "id" => &phab, + stamp, + reason, + actor, + "comment" => &plain_body, + }).map_err(|e| format!("{:?}", e))?; + if result.affected_rows() != 1 { + return Err(format!("Affected row count for comments was {}, not 1", result.affected_rows())); + } + } else { + db.prep_exec(r#"INSERT INTO metadata (bug, stamp, title, secure, note) + VALUES (:id, FROM_UNIXTIME(:stamp), :title, :secure, "") + ON DUPLICATE KEY UPDATE stamp=VALUES(stamp), title=VALUES(title), secure=VALUES(secure)"#, params! { + "id" => &phab, + stamp, + "title" => &title, + "secure" => false, + }).map_err(|e| format!("{:?}", e))?; + let result = db.prep_exec(r#"INSERT INTO phab_diffs (revision, stamp, reason, author, comment) + VALUES (:revision, FROM_UNIXTIME(:stamp), :reason, :actor, :comment)"#, params! { + "revision" => &phab, + stamp, + reason, + actor, + "comment" => &plain_body, + }).map_err(|e| format!("{:?}", e))?; + if result.affected_rows() != 1 { + return Err(format!("Affected row count for phab_diffs was {}, not 1", result.affected_rows())); + } } Ok(()) diff --git a/www/search.php b/www/search.php index 21267dc..3c26b6d 100644 --- a/www/search.php +++ b/www/search.php @@ -44,7 +44,8 @@ 'changes' => array( 'field', 'oldval', 'newval' ), 'comments' => array( 'author', 'comment' ), 'newbugs' => array( 'title', 'author', 'description' ), - 'metadata' => array( 'title' ) + 'metadata' => array( 'title' ), + 'gh_issues' => array( 'author', 'comment' ), ); function lengthSort( $a, $b ) { From c305a002ee3716f83c6255acfd4d898ec1de621a Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 7 Feb 2022 01:26:41 +0000 Subject: [PATCH 137/141] Deal with slight change to github email that caused slashes to appear in subsequent lines of the footer --- scraper/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scraper/src/main.rs b/scraper/src/main.rs index d9f8520..d31e5e2 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -96,8 +96,8 @@ fn url_parts(footer: String) -> Option<(String, String, Option)> { let issues_ix = issues_ix? + repo_ix; let issues_len = issues_len.unwrap(); let issuenum_ix = issues_ix + issues_len; - let hash_ix = footer[issuenum_ix..].find(hash).or_else(|| footer[issuenum_ix..].find(slash)).map(|ix| ix + issuenum_ix); let end_ix = footer[repo_ix..].find("\n").map(|ix| ix + repo_ix).unwrap_or(footer.len()); + let hash_ix = footer[issuenum_ix..end_ix].find(hash).or_else(|| footer[issuenum_ix..end_ix].find(slash)).map(|ix| ix + issuenum_ix); let hash = match hash_ix { Some(ix) => Some(String::from(footer[ix..end_ix].trim())), None => None, From 8f794cf69812d54976aecd13822bb6cfac6ed58a Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sat, 2 Mar 2024 16:17:23 +0000 Subject: [PATCH 138/141] Explicit row format for easier migration across mysql versions --- schemas/tables.sql | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/schemas/tables.sql b/schemas/tables.sql index e20ed68..0307a18 100644 --- a/schemas/tables.sql +++ b/schemas/tables.sql @@ -11,7 +11,7 @@ CREATE TABLE `requests` ( KEY (`bug`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; CREATE TABLE `reviews` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -29,7 +29,7 @@ CREATE TABLE `reviews` ( KEY (`bug`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; CREATE TABLE `changes` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -44,7 +44,7 @@ CREATE TABLE `changes` ( KEY (`bug`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; CREATE TABLE `comments` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -59,7 +59,7 @@ CREATE TABLE `comments` ( KEY (`bug`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; CREATE TABLE `newbugs` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -74,7 +74,7 @@ CREATE TABLE `newbugs` ( KEY (`bug`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; CREATE TABLE `gh_issues` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -90,7 +90,7 @@ CREATE TABLE `gh_issues` ( KEY (`repo`, `issue`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; CREATE TABLE `phab_diffs` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -104,7 +104,7 @@ CREATE TABLE `phab_diffs` ( KEY (`revision`), KEY (`stamp`), KEY (`viewed`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; CREATE TABLE `metadata` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -112,10 +112,10 @@ CREATE TABLE `metadata` ( `stamp` datetime NOT NULL, `title` varchar(1024) NOT NULL DEFAULT '', `secure` tinyint(1) NOT NULL DEFAULT 0, - `note` mediumtext NOT NULL DEFAULT '', + `note` mediumtext NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY (`bug`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; CREATE TABLE `tags` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, @@ -124,4 +124,4 @@ CREATE TABLE `tags` ( PRIMARY KEY (`id`), KEY (`bug`), KEY (`tag`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; From da596d0fe7383c9f51148c722374cf2028b28ad7 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Sat, 2 Mar 2024 16:17:47 +0000 Subject: [PATCH 139/141] php 8 fixes --- www/common.php | 14 +------------- www/dashboard.php | 6 +++--- www/wipe.php | 2 +- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/www/common.php b/www/common.php index 58fc5b4..6117865 100644 --- a/www/common.php +++ b/www/common.php @@ -1,17 +1,5 @@ $value) { - $_POST[$key] = stripslashes($value); - } - foreach ($_GET AS $key => $value) { - $_GET[$key] = stripslashes($value); - } - foreach ($_COOKIE AS $key => $value) { - $_COOKIE[$key] = stripslashes($value); - } -} - function fail( $message ) { error_log( $message ); header( 'HTTP/500 Error!' ); @@ -66,7 +54,7 @@ function isGithubCommit( $bugid ) { } function isPhabDiff( $bugid ) { - return ($bugid[0] == 'D'); + return (strval($bugid)[0] == 'D'); } function makeBugLink( $bugid ) { diff --git a/www/dashboard.php b/www/dashboard.php index 450d3e3..1336e89 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -107,8 +107,8 @@ function buglink( $prefix, $bug ) { function linkify( $text, $bug ) { global $_BASE_URL; $text = preg_replace( '#(https?://\S+)#i', '$1', $text ); - $text = preg_replace( '/(bug\s+)(\d+)/ie', 'buglink(\'\\1\', \'\\2\')', $text ); - $text = preg_replace( '/(bug-)(\d+)/ie', 'buglink(\'\\1\', \'\\2\')', $text ); + $text = preg_replace_callback( '/(bug\s+)(\d+)/i', function($m) { return buglink($m[1], $m[2]); }, $text ); + $text = preg_replace_callback( '/(bug-)(\d+)/i', function($m) { return buglink($m[1], $m[2]); }, $text ); $text = preg_replace( '/(Attachment #?)(\d+)/i', '$1$2', $text ); return $text; } @@ -129,7 +129,7 @@ function linkify_phab( $text ) { function buglinkify( $field, $text ) { global $_BASE_URL; if ($field === 'Depends on' || $field === 'Blocks' || $field === 'Regressions' || $field === 'Regressed by') { - $text = preg_replace( '/(\d+)/e', 'buglink(\'\', \'\\1\')', $text ); + $text = preg_replace_callback( '/(\d+)/', function($m) { return buglink('', $m[1]); }, $text ); } return $text; } diff --git a/www/wipe.php b/www/wipe.php index 9b539f0..9e4d5ad 100644 --- a/www/wipe.php +++ b/www/wipe.php @@ -13,7 +13,7 @@ $ids = explode( ',', $_POST['ids'] ); foreach ($ids AS $id) { - switch ($id{0}) { + switch ($id[0]) { case 'r': $table = 'reviews'; break; From aa4d1150bf9a4457eea8e3f49b15c6d2241ebe5a Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 15 Apr 2024 03:49:08 +0000 Subject: [PATCH 140/141] Add http support for certbot --- nginx/nginx.conf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 5b8f9e3..c8f2a51 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -2,7 +2,15 @@ server { listen [::]:80; listen 80; server_name bugmash.staktrace.com; - return 301 https://$host$request_uri; + root /ebs/bugmash/www; + + location /.well-known { + try_files $uri $uri/ =404; + } + + location / { + return 301 https://$host$request_uri; + } } server { From 3aaca223c9308e88d46586a18bd77ef7da7a0c1a Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Mon, 20 May 2024 01:36:32 +0000 Subject: [PATCH 141/141] Drop spurious slash in URL causing 404s --- www/dashboard.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/dashboard.php b/www/dashboard.php index 1336e89..b9bc9c1 100644 --- a/www/dashboard.php +++ b/www/dashboard.php @@ -317,7 +317,7 @@ function safeGet( &$array, $index ) { $bugid = $row['repo'] . '#' . $row['issue']; initEmpty( $bblocks, $bugid, $stamp ); if ($row['hash'] == "") { - $bblocks[ $bugid ][ $stamp ] .= sprintf( '
New: %s by %s
%s
', + $bblocks[ $bugid ][ $stamp ] .= sprintf( '
New: %s by %s
%s
', $row['id'], $_GH_BASE_URL, $row['repo'],