Add referers to logging.

lib/TCMS.pm

@@ -30,6 +30,8 @@ use Trog::Routes::HTML;
 use Trog::Routes::JSON;
 
 use Trog::Log qw{:all};
+use Trog::Log::DBI;
+
 use Trog::Auth;
 use Trog::Utils;
 use Trog::Config;
@@ -124,7 +126,9 @@ sub _app {
     # These two parameters are entirely academic, as no integration with any kind of analytics is implemented.
     #my $no_track = $env->{HTTP_DNT};
     #my $no_sell_info = $env->{HTTP_SEC_GPC};
-    #my $referrer     = $env->{HTTP_REFERER};
+
+    # Set the referer to go into DB logs, but not logs in general.
+    $Trog::Log::DBI::referer = $env->{HTTP_REFERER};
 
     # We generally prefer this to be handled at the reverse proxy level.
     #my $prefer_ssl = $env->{HTTP_UPGRADE_INSECURE_REQUESTS};

lib/Trog/Log.pm

@@ -21,8 +21,7 @@ $LOGNAME = $ENV{CUSTOM_LOG} if $ENV{CUSTOM_LOG};
 
 my $LEVEL = $ENV{WWW_VERBOSE} ? 'debug' : 'info';
 
-our $log;
-our $user;
+our ($log, $user);
 $Trog::Log::user = 'nobody';
 $Trog::Log::ip   = '0.0.0.0';
 

lib/Trog/Log/DBI.pm

@@ -8,11 +8,13 @@ use parent qw{Log::Dispatch::DBI};
 use Ref::Util qw{is_arrayref};
 use Capture::Tiny qw{capture_merged};
 
+our $referer;
+
 sub create_statement {
     my $self = shift;
 
     # This is a writable view.  Consult schema for its behavior.
-    my $sql = "INSERT INTO all_requests (uuid, date, ip_address, user, method, route, code) VALUES (?,?,?,?,?,?,?)";
+    my $sql = "INSERT INTO all_requests (uuid, date, ip_address, user, method, route, code, referer) VALUES (?,?,?,?,?,?,?,?)";
 
     my $sql2 = "INSERT INTO messages (uuid, message) VALUES (?,?)";
     $self->{sth2} = $self->{dbh}->prepare($sql2);
@@ -42,7 +44,11 @@ sub log_message {
     # If this is a mangled log, forget it.
     return unless $date && $uuid;
 
-    my $res = $self->{sth}->execute($uuid, $date, $ip, $user, $method, $route, $code );
+    # Allow callers to set referer.
+    # We only care about this in DB context, as it's only for metrics, which are irrelevant in text logs/debugging.
+    $referer //= 'none';
+
+    my $res = $self->{sth}->execute($uuid, $date, $ip, $user, $method, $route, $code, $referer );
 
     if (is_arrayref($buffer{$uuid}) && @{$buffer{$uuid}}) {
         $self->{sth2}->bind_param_array(1, $uuid);

schema/log.schema

@@ -20,12 +20,18 @@ CREATE TABLE IF NOT EXISTS response_code (
     code INTEGER NOT NULL UNIQUE
 );
 
+CREATE TABLE IF NOT EXISTS referer (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    referer TEXT NOT NULL UNIQUE
+);
+
 CREATE TABLE IF NOT EXISTS requests (
     uuid TEXT PRIMARY KEY,
     date TEXT NOT NULL,
     host_id INTEGER NOT NULL REFERENCES seen_hosts(id) ON DELETE CASCADE,
     user_id INTEGER NOT NULL REFERENCES seen_users(id) ON DELETE CASCADE,
     route_id INTEGER NOT NULL REFERENCES seen_routes(id) ON DELETE CASCADE,
+    referer_id INTEGER NOT NULL REFERENCES referer(id) ON DELETE CASCADE,
     response_code_id INTEGER NOT NULL REFERENCES response_code(id) ON DELETE RESTRICT
 );
 
@@ -37,6 +43,7 @@ CREATE VIEW IF NOT EXISTS all_requests AS
         u.user,
         r.method,
         r.route,
+        f.referer,
         c.code
     FROM
         requests AS q
@@ -46,25 +53,30 @@ CREATE VIEW IF NOT EXISTS all_requests AS
         seen_users AS u ON q.user_id = u.id
     JOIN
         seen_routes AS r ON q.route_id = r.id
+    JOIN
+        referer AS f ON q.referer_id = f.id
     JOIN
         response_code AS c on q.response_code_id = c.id;
 
 /* Make all_requests a writable view via triggers.  We will always stomp the main row, as the last update will be what we want. */
 CREATE TRIGGER IF NOT EXISTS insert_all_requests INSTEAD OF INSERT ON all_requests BEGIN
-    INSERT OR IGNORE  INTO response_code (code)         VALUES (NEW.code);
-    INSERT OR IGNORE  INTO seen_routes   (route,method) VALUES (NEW.route, NEW.method);
-    INSERT OR IGNORE  INTO seen_users    (user)         VALUES (NEW.user);
-    INSERT OR IGNORE  INTO seen_hosts    (ip_address)   VALUES (NEW.ip_address);
+    INSERT OR IGNORE INTO response_code (code)         VALUES (NEW.code);
+    INSERT OR IGNORE INTO seen_routes   (route,method) VALUES (NEW.route, NEW.method);
+    INSERT OR IGNORE INTO seen_users    (user)         VALUES (NEW.user);
+    INSERT OR IGNORE INTO seen_hosts    (ip_address)   VALUES (NEW.ip_address);
+    INSERT OR IGNORE INTO referer       (referer)      VALUES (NEW.referer);
     INSERT OR REPLACE INTO requests SELECT
         NEW.uuid,
         NEW.date,
         h.id AS host_id,
         u.id AS user_id,
         r.id AS route_id,
+        f.id AS referer_id,
         c.id AS response_code_id
     FROM seen_hosts AS h
     JOIN seen_users AS u ON u.user = NEW.user
     JOIN seen_routes AS r ON r.route = NEW.route AND r.method = NEW.method
+    JOIN referer AS f ON f.referer = NEW.referer
     JOIN response_code AS c ON c.code = NEW.code
     WHERE h.ip_address = NEW.ip_address;
 END;