[Summary view] [Print] [Text view]

   1  #!/usr/bin/php
   2  <?php
   3  // $Id: project-usage-process.php,v 1.7 2009/08/07 15:42:32 dww Exp $
   4  
   5  
   6  /**
   7   * @file
   8   * Processes the project_usage statistics.
   9   *
  10   * @author Andrew Morton (http://drupal.org/user/34869)
  11   * @author Derek Wright (http://drupal.org/user/46549)
  12   *
  13   */
  14  
  15  // ------------------------------------------------------------
  16  // Required customization
  17  // ------------------------------------------------------------
  18  
  19  // The root of your Drupal installation, so we can properly bootstrap
  20  // Drupal. This should be the full path to the directory that holds
  21  // your index.php file, the "includes" subdirectory, etc.
  22  define('DRUPAL_ROOT', '');
  23  
  24  // The name of your site. Required so that when we bootstrap Drupal in
  25  // this script, we find the right settings.php file in your sites folder.
  26  define('SITE_NAME', '');
  27  
  28  
  29  // ------------------------------------------------------------
  30  // Initialization
  31  // (Real work begins here, nothing else to customize)
  32  // ------------------------------------------------------------
  33  
  34  // Check if all required variables are defined
  35  $vars = array(
  36    'DRUPAL_ROOT' => DRUPAL_ROOT,
  37    'SITE_NAME' => SITE_NAME,
  38  );
  39  $fatal_err = FALSE;
  40  foreach ($vars as $name => $val) {
  41    if (empty($val)) {
  42      print "ERROR: \"$name\" constant not defined, aborting\n";
  43      $fatal_err = TRUE;
  44    }
  45  }
  46  if ($fatal_err) {
  47    exit(1);
  48  }
  49  
  50  $script_name = $argv[0];
  51  
  52  // Setup variables for Drupal bootstrap
  53  $_SERVER['HTTP_HOST'] = SITE_NAME;
  54  $_SERVER['REMOTE_ADDR'] = '127.0.0.1';
  55  $_SERVER['REQUEST_URI'] = '/' . $script_name;
  56  $_SERVER['SCRIPT_NAME'] = '/' . $script_name;
  57  $_SERVER['PHP_SELF'] = '/' . $script_name;
  58  $_SERVER['SCRIPT_FILENAME'] = $_SERVER['PWD'] .'/'. $script_name;
  59  $_SERVER['PATH_TRANSLATED'] = $_SERVER['SCRIPT_FILENAME'];
  60  
  61  if (!chdir(DRUPAL_ROOT)) {
  62    print "ERROR: Can't chdir(DRUPAL_ROOT), aborting.\n";
  63    exit(1);
  64  }
  65  // Make sure our umask is sane for generating directories and files.
  66  umask(022);
  67  
  68  require_once  'includes/bootstrap.inc';
  69  drupal_bootstrap(DRUPAL_BOOTSTRAP_FULL);
  70  
  71  if (!module_exists('project_usage')) {
  72    wd_err(t("ERROR: Project usage module does not exist, aborting.\n"));
  73    exit(1);
  74  }
  75  
  76  // Load the API functions we need for manipulating dates and timestamps.
  77  module_load_include('inc', 'project_usage', 'includes/date_api');
  78  
  79  // ------------------------------------------------------------
  80  // Call the daily and weekly processing tasks as needed.
  81  // ------------------------------------------------------------
  82  
  83  $now = time();
  84  
  85  // Figure out if it's been 24 hours since our last daily processing.
  86  if (variable_get('project_usage_last_daily', 0) <= ($now - PROJECT_USAGE_DAY)) {
  87    project_usage_process_daily();
  88    variable_set('project_usage_last_daily', $now);
  89  }
  90  
  91  // We can't process the weekly data until the week has completed. To see if
  92  // there's data available: determine the last time we completed the weekly
  93  // processing and compare that to the start of this week. If the last
  94  // weekly processing occurred before the current week began then there should
  95  // be one (or more) week's worth of data ready to process.
  96  $default = $now - variable_get('project_usage_life_daily', 4 * PROJECT_USAGE_WEEK);
  97  $last_weekly = variable_get('project_usage_last_weekly', $default);
  98  $current_week_start = project_usage_weekly_timestamp(NULL, 0);
  99  if ($last_weekly <= $current_week_start) {
 100    project_usage_process_weekly($last_weekly);
 101    variable_set('project_usage_last_weekly', $now);
 102    // Reset the list of active weeks.
 103    project_usage_get_active_weeks(TRUE);
 104  }
 105  
 106  // Wipe the cache of all expired usage pages.
 107  cache_clear_all(NULL, 'cache_project_usage');
 108  
 109  
 110  /**
 111   * Process all the raw data up to the previous day.
 112   *
 113   * The primary key on the {project_usage_raw} table will prevent duplicate
 114   * records provided we process them once the day is complete. If we pull them
 115   * out too soon and the site checks in again they will be counted twice.
 116   */
 117  function project_usage_process_daily() {
 118    // Timestamp for begining of the previous day.
 119    $timestamp = project_usage_daily_timestamp(NULL, 1);
 120    $time_0 = time();
 121  
 122    watchdog('project_usage', 'Starting to process daily usage data for !date.', array('!date' => format_date($timestamp, 'custom', 'Y-m-d')));
 123  
 124    // Assign API version term IDs.
 125    $terms = array();
 126    foreach (project_release_get_api_taxonomy() as $term) {
 127      $terms[$term->tid] = $term->name;
 128    }
 129    $num_updates = 0;
 130    $query = db_query("SELECT DISTINCT api_version FROM {project_usage_raw} WHERE tid = 0");
 131    while ($row = db_fetch_object($query)) {
 132      $tid = array_search($row->api_version, $terms);
 133      db_query("UPDATE {project_usage_raw} SET tid = %d WHERE api_version = '%s'", $tid, $row->api_version);
 134      $num_updates += db_affected_rows();
 135    }
 136    $time_1 = time();
 137    $substitutions = array(
 138      '!rows' => format_plural($num_updates, '1 row', '@count rows'),
 139      '!delta' => format_interval($time_1 - $time_0),
 140    );
 141    watchdog('project_usage', 'Assigned API version term IDs for !rows (!delta).', $substitutions);
 142  
 143    // Asign project and release node IDs.
 144    $num_updates = 0;
 145    $query = db_query("SELECT DISTINCT project_uri, project_version FROM {project_usage_raw} WHERE pid = 0 OR nid = 0");
 146    while ($row = db_fetch_object($query)) {
 147      $pid = db_result(db_query("SELECT pp.nid AS pid FROM {project_projects} pp WHERE pp.uri = '%s'", $row->project_uri));
 148      if ($pid) {
 149        $nid = db_result(db_query("SELECT prn.nid FROM {project_release_nodes} prn WHERE prn.pid = %d AND prn.version = '%s'", $pid, $row->project_version));
 150        db_query("UPDATE {project_usage_raw} SET pid = %d, nid = %d WHERE project_uri = '%s' AND project_version = '%s'", $pid, $nid, $row->project_uri, $row->project_version);
 151        $num_updates += db_affected_rows();
 152      }
 153    }
 154    $time_2 = time();
 155    $substitutions = array(
 156      '!rows' => format_plural($num_updates, '1 row', '@count rows'),
 157      '!delta' => format_interval($time_2 - $time_1),
 158    );
 159    watchdog('project_usage', 'Assigned project and release node IDs to !rows (!delta).', $substitutions);
 160  
 161    // Move usage records with project node IDs into the daily table and remove
 162    // the rest.
 163    db_query("INSERT INTO {project_usage_day} (timestamp, site_key, pid, nid, tid, ip_addr) SELECT timestamp, site_key, pid, nid, tid, ip_addr FROM {project_usage_raw} WHERE timestamp < %d AND pid <> 0", $timestamp);
 164    $num_new_day_rows = db_affected_rows();
 165    db_query("DELETE FROM {project_usage_raw} WHERE timestamp < %d", $timestamp);
 166    $num_deleted_raw_rows = db_affected_rows();
 167    $time_3 = time();
 168    $substitutions = array(
 169      '!day_rows' => format_plural($num_new_day_rows, '1 row', '@count rows'),
 170      '!raw_rows' => format_plural($num_deleted_raw_rows, '1 row', '@count rows'),
 171      '!delta' => format_interval($time_3 - $time_2),
 172    );
 173    watchdog('project_usage', 'Moved usage from raw to daily: !day_rows added to {project_usage_day}, !raw_rows deleted from {project_usage_raw} (!delta).', $substitutions);
 174  
 175    // Remove old daily records.
 176    $seconds = variable_get('project_usage_life_daily', 4 * PROJECT_USAGE_WEEK);
 177    db_query("DELETE FROM {project_usage_day} WHERE timestamp < %d", time() - $seconds);
 178    $time_4 = time();
 179    $substitutions = array(
 180      '!rows' => format_plural(db_affected_rows(), '1 old daily row', '@count old daily rows'),
 181      '!delta' => format_interval($time_4 - $time_3),
 182    );
 183    watchdog('project_usage', 'Removed !rows (!delta).', $substitutions);
 184  
 185    watchdog('project_usage', 'Completed daily usage data processing (total time: !delta).', array('!delta' => format_interval($time_4 - $time_0)));
 186  }
 187  
 188  /**
 189   * Compute the weekly summaries for the week starting at the given timestamp.
 190   *
 191   * @param $timestamp
 192   *   UNIX timestamp indicating the last time weekly stats were processed.
 193   */
 194  function project_usage_process_weekly($timestamp) {
 195    watchdog('project_usage', 'Starting to process weekly usage data.');
 196    $time_0 = time();
 197  
 198    // Get all the weeks since we last ran.
 199    $weeks = project_usage_get_weeks_since($timestamp);
 200    // Skip the last entry since it's the current, incomplete week.
 201    $count = count($weeks) - 1;
 202    for ($i = 0; $i < $count; $i++) {
 203      $start = $weeks[$i];
 204      $end = $weeks[$i + 1];
 205      $date = format_date($start, 'custom', 'Y-m-d');
 206      $time_1 = time();
 207  
 208      // Try to compute the usage tallies per project and per release. If there
 209      // is a problem--perhaps some rows existed from a previous, incomplete
 210      // run that are preventing inserts, throw a watchdog error.
 211  
 212      $sql = "INSERT INTO {project_usage_week_project} (nid, timestamp, tid, count) SELECT pid, %d, tid, COUNT(DISTINCT site_key) FROM {project_usage_day} WHERE timestamp >= %d AND timestamp < %d AND pid <> 0 GROUP BY pid, tid";
 213      $query_args = array($start, $start, $end);
 214      $result = db_query($sql, $query_args);
 215      $time_2 = time();
 216      _db_query_callback($query_args, TRUE);
 217      $substitutions = array(
 218        '!date' => $date,
 219        '%query' => preg_replace_callback(DB_QUERY_REGEXP, '_db_query_callback', $sql),
 220        '!projects' => format_plural(db_affected_rows(), '1 project', '@count projects'),
 221        '!delta' => format_interval($time_2 - $time_1),
 222      );
 223      if (!$result) {
 224        watchdog('project_usage', 'Query failed inserting weekly project tallies for !date, query: %query (!delta).', $substitutions, WATCHDOG_ERROR);
 225      }
 226      else {
 227        watchdog('project_usage', 'Computed weekly project tallies for !date for !projects (!delta).', $substitutions);
 228      }
 229  
 230      $sql = "INSERT INTO {project_usage_week_release} (nid, timestamp, count) SELECT nid, %d, COUNT(DISTINCT site_key) FROM {project_usage_day} WHERE timestamp >= %d AND timestamp < %d AND nid <> 0 GROUP BY nid";
 231      $query_args = array($start, $start, $end);
 232      $result = db_query($sql, $query_args);
 233      $time_3 = time();
 234      _db_query_callback($query_args, TRUE);
 235      $substitutions = array(
 236        '!date' => $date,
 237        '!releases' => format_plural(db_affected_rows(), '1 release', '@count releases'),
 238        '%query' => preg_replace_callback(DB_QUERY_REGEXP, '_db_query_callback', $sql),
 239        '!delta' => format_interval($time_3 - $time_2),
 240      );
 241      if (!$result) {
 242        watchdog('project_usage', 'Query failed inserting weekly release tallies for !date, query: %query (!delta).', $substitutions, WATCHDOG_ERROR);
 243      }
 244      else {
 245        watchdog('project_usage', 'Computed weekly release tallies for !date for !releases (!delta).', $substitutions);
 246      }
 247    }
 248  
 249    // Remove any tallies that have aged out.
 250    $time_4 = time();
 251    $project_life = variable_get('project_usage_life_weekly_project', PROJECT_USAGE_YEAR);
 252    db_query("DELETE FROM {project_usage_week_project} WHERE timestamp < %d", $now - $project_life);
 253    $time_5 = time();
 254    $substitutions = array(
 255      '!rows' => format_plural(db_affected_rows(), '1 old weekly project row', '@count old weekly project rows'),
 256      '!delta' => format_interval($time_5 - $time_4),
 257    );
 258    watchdog('project_usage', 'Removed !rows (!delta).', $substitutions);
 259  
 260    $release_life = variable_get('project_usage_life_weekly_release', 26 * PROJECT_USAGE_WEEK);
 261    db_query("DELETE FROM {project_usage_week_release} WHERE timestamp < %d", $now - $release_life);
 262    $time_6 = time();
 263    $substitutions = array(
 264      '!rows' => format_plural(db_affected_rows(), '1 old weekly release row', '@count old weekly release rows'),
 265      '!delta' => format_interval($time_6 - $time_5),
 266    );
 267    watchdog('project_usage', 'Removed !rows (!delta).', $substitutions);
 268  
 269    watchdog('project_usage', 'Completed weekly usage data processing (total time: !delta).', array('!delta' => format_interval($time_6 - $time_0)));
 270  }
 271