Hackfest, implement frontend

This commit is contained in:
Mikko Ahlroth 2019-11-03 10:54:04 +02:00
parent 088bd1f532
commit e2b74cfa9d
25 changed files with 800 additions and 4 deletions

View file

@ -0,0 +1,53 @@
defmodule Tilastokeskus.Archive.Archivist do
@moduledoc """
The archivist is responsible for updating clients about updates to the archive. When new data
is inserted, the archivist should be notified. It will then notify every client waiting for
new information.
"""
alias Tilastokeskus.Archive.Schemas.PageView
@doc """
Get the name to use for the registry process.
"""
@spec registry_name() :: atom
def registry_name(), do: __MODULE__
@doc """
Get name to use for registry key when listening to updates on the given host.
"""
@spec listen_key(String.t()) :: {atom, String.t()}
def listen_key(host) when is_binary(host), do: {:archivist_listener, host}
@doc """
Return child tuple to add to supervisor to start archivist. If name is not given,
`registry_name/0` is used.
"""
@spec child_tuple(atom) :: {module, list}
def child_tuple(name \\ nil) do
{Registry,
[
name: if(not is_nil(name), do: name, else: registry_name()),
keys: :duplicate,
partitions: System.schedulers_online()
]}
end
@doc """
Register to listen for updates to a given host.
"""
@spec register(String.t()) :: {:ok, pid} | {:error, any}
def register(host) when is_binary(host) do
Registry.register(registry_name(), listen_key(host), [])
end
@doc """
Send a page view update to any applicable listeners.
"""
@spec update(PageView.t()) :: :ok
def update(%PageView{} = view) do
Registry.dispatch(registry_name(), listen_key(view.host), fn entries ->
for {pid, _} <- entries, do: send(pid, {:archivist_update, view})
end)
end
end

View file

@ -45,6 +45,8 @@ defmodule Tilastokeskus.Archive.Schemas.PageView do
# GeoIP calculated information # GeoIP calculated information
field(:loc_city, :string) field(:loc_city, :string)
field(:loc_country, :string) field(:loc_country, :string)
field(:loc_lat, :float)
field(:loc_lon, :float)
# Was the hit from a bot # Was the hit from a bot
field(:is_bot, :boolean) field(:is_bot, :boolean)
@ -77,6 +79,8 @@ defmodule Tilastokeskus.Archive.Schemas.PageView do
:tz_offset, :tz_offset,
:loc_city, :loc_city,
:loc_country, :loc_country,
:loc_lat,
:loc_lon,
:is_bot :is_bot
]) ])
|> put_change(:type, event_type()) |> put_change(:type, event_type())

View file

@ -47,7 +47,9 @@ defmodule Tilastokeskus.Archive.Scrubinator do
scrubbed: true, scrubbed: true,
addr: nil, addr: nil,
ua: nil, ua: nil,
loc_city: nil loc_city: nil,
loc_lat: 0.0,
loc_lon: 0.0
] ]
] ]
) )

View file

@ -3,14 +3,37 @@ defmodule Tilastokeskus.Archive.Utils.PageView do
Pageview related utilities. Pageview related utilities.
""" """
import Ecto.Query, only: [from: 2]
alias Tilastokeskus.Archive.Repo alias Tilastokeskus.Archive.Repo
alias Tilastokeskus.Archive.Schemas.PageView alias Tilastokeskus.Archive.Schemas.PageView
@doc """ @doc """
Create new pageview from changeset. Create new pageview from changeset.
""" """
@spec create(Ecto.Changeset.t()) :: {:ok, %PageView{}} | {:error, term} @spec create(Ecto.Changeset.t()) :: {:ok, PageView.t()} | {:error, term}
def create(changeset) do def create(changeset) do
Repo.insert(changeset) case Repo.insert(changeset) do
{:ok, %PageView{} = view} ->
Tilastokeskus.Archive.Archivist.update(view)
{:ok, view}
e ->
e
end
end
@doc """
Get page views for the last `mins` minutes for the given host.
"""
@spec get_last(String.t(), integer) :: [PageView.t()]
def get_last(host, mins) do
now = DateTime.utc_now()
then = DateTime.add(now, -(mins * 60))
from(pw in PageView,
where: pw.host == ^host and pw.at >= ^then
)
|> Repo.all()
end end
end end

View file

@ -3,6 +3,7 @@ defmodule Tilastokeskus.Reception.Router do
section([{Raxx.Logger, level: :debug}], [ section([{Raxx.Logger, level: :debug}], [
{%{method: :POST, path: ["track"]}, Tilastokeskus.Reception.Routes.PageView}, {%{method: :POST, path: ["track"]}, Tilastokeskus.Reception.Routes.PageView},
{%{method: :GET, path: ["live", _host]}, Tilastokeskus.Reception.Routes.LiveView},
{_, Tilastokeskus.Reception.Routes.NotFound} {_, Tilastokeskus.Reception.Routes.NotFound}
]) ])
end end

View file

@ -0,0 +1,96 @@
defmodule Tilastokeskus.Reception.Routes.LiveView do
use Raxx.Server
alias Tilastokeskus.Archive.Schemas.PageView
@heartbeat_delay 30_000
@initial_data_mins 5
@typep req_state :: nil
@typep resp :: {[Raxx.Response.t()], req_state}
@impl Raxx.Server
@spec handle_head(Raxx.Request.t(), any) :: resp
def handle_head(%{method: :GET, path: ["live", host]}, _state) do
resp =
response(200)
|> set_header("content-type", "text/event-stream")
|> set_header("access-control-allow-origin", "*")
|> set_body(true)
# Register to listen for live updates
Tilastokeskus.Archive.Archivist.register(host)
# Start heartbeat ping
start_heartbeat_timer()
# Send initial data ASAP
Process.send_after(self(), {:init, host}, 0)
{[resp], nil}
end
@spec handle_info(any, req_state) :: resp
def handle_info(msg, state)
def handle_info({:archivist_update, %PageView{} = view}, state) do
{[view |> transform_view() |> data()], state}
end
def handle_info(:heartbeat, state) do
# Hearbeat, why do you miss when my baby kisses me?
start_heartbeat_timer()
{[data(":ping\n\n")], state}
end
def handle_info({:init, host}, state) do
# Get last minutes' data immediately
immediate_data =
Tilastokeskus.Archive.Utils.PageView.get_last(host, @initial_data_mins)
|> Enum.map(&transform_view/1)
|> Enum.join("")
immediate_data = if immediate_data == "", do: ":nil", else: immediate_data
{[data(immediate_data)], state}
end
def handle_info(_, state), do: {[], state}
# Heartbeat, why does a love kiss stay in my memory?
defp start_heartbeat_timer(), do: Process.send_after(self(), :heartbeat, @heartbeat_delay)
# Transform view to SSE event
defp transform_view(view) do
content =
Jason.encode!(%{
at: view.at,
addr: view.addr |> :inet_parse.ntoa() |> to_string(),
extra: view.extra,
scrubbed: view.scrubbed,
session: view.session_id,
path: view.path,
path_noq: view.path_noq,
host: view.host,
referrer: view.referrer,
referrer_domain: view.referrer_domain,
ua: view.ua,
ua_name: view.ua_name,
ua_version: view.ua_version,
os_name: view.os_name,
os_version: view.os_version,
device_type: view.device_type,
screen_w: view.screen_w,
screen_h: view.screen_h,
tz_offset: view.tz_offset,
loc_city: view.loc_city,
loc_country: view.loc_country,
loc_lat: view.loc_lat,
loc_lon: view.loc_lon,
is_bot: view.is_bot
})
content_data_rows = String.split(content, "\n") |> Enum.join("\ndata: ")
"event: view\ndata: #{content_data_rows}\n\n"
end
end

View file

@ -23,7 +23,10 @@ defmodule Tilastokeskus.Reception.Routes.PageView do
screen_h = Map.get(body, "screen_height") screen_h = Map.get(body, "screen_height")
tz_offset = Map.get(body, "tz_offset") tz_offset = Map.get(body, "tz_offset")
{city, country} = get_geoip(addr) {city, {lat, lon}, country} = get_geoip(addr)
lat = if not is_nil(lat), do: lat, else: 0.0
lon = if not is_nil(lon), do: lon, else: 0.0
# Run in one transaction to avoid multiple DB checkouts # Run in one transaction to avoid multiple DB checkouts
{:ok, response} = {:ok, response} =
@ -54,6 +57,8 @@ defmodule Tilastokeskus.Reception.Routes.PageView do
tz_offset: tz_offset, tz_offset: tz_offset,
loc_city: city, loc_city: city,
loc_country: country, loc_country: country,
loc_lat: lat,
loc_lon: lon,
is_bot: is_bot is_bot: is_bot
} }
) )
@ -186,6 +191,7 @@ defmodule Tilastokeskus.Reception.Routes.PageView do
{ {
get_in(city, [:city, :names, :en]), get_in(city, [:city, :names, :en]),
{get_in(city, [:location, :latitude]), get_in(city, [:location, :longitude])},
get_in(country, [:country, :names, :en]) get_in(country, [:country, :names, :en])
} }
end end

View file

@ -14,6 +14,8 @@ defmodule Tilastokeskus.Application do
# List all child processes to be supervised # List all child processes to be supervised
children = [ children = [
# PubSub system for live updates using registry
Tilastokeskus.Archive.Archivist.child_tuple(),
{Tilastokeskus.Archive.Repo, []}, {Tilastokeskus.Archive.Repo, []},
{Tilastokeskus.Archive.Scrubinator, %{days: days}}, {Tilastokeskus.Archive.Scrubinator, %{days: days}},
{Tilastokeskus.Reception.FrontDesk, [hosts: hosts, port: port, cleartext: true]} {Tilastokeskus.Reception.FrontDesk, [hosts: hosts, port: port, cleartext: true]}

View file

@ -0,0 +1,10 @@
defmodule Tilastokeskus.Archive.Repo.Migrations.AddLatLon do
use Ecto.Migration
def change do
alter table(:events) do
add(:loc_lat, :"double precision", null: false, default: 0.0)
add(:loc_lon, :"double precision", null: false, default: 0.0)
end
end
end

2
frontend/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
node_modules
build

35
frontend/index.html Normal file
View file

@ -0,0 +1,35 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>Tilastokeskus</title>
<link rel="stylesheet" href="style.css" />
</head>
<body>
<main>
<h1>Tilastokeskus</h1>
<section id="currently-live">
<div id="currently-live-counter">?</div>
<div>users currently live</div>
</section>
<section id="top-paths">
<h2>Top paths</h2>
<div id="top-paths-data"></div>
</section>
<section id="world-map"></section>
<section id="current-sessions">
<h2>Latest pageviews</h2>
<div id="current-sessions-data"></div>
</section>
</main>
<script type="module" src="build/index.js" async></script>
</body>
</html>

View file

@ -0,0 +1,37 @@
const ARCHIVE_BASE_URL = "http://localhost:1971/live";
const PAGE_VIEW_EVENT = "view";
/**
* Get the URL path to use for updates about the given host.
*/
function hostUpdateUrl(host: string): string {
return `${ARCHIVE_BASE_URL}/${host}`;
}
/**
* Start listening for updates to the given host.
*/
export function connectListener(
host: string,
openCallback: (stream: EventSource, evt: Event) => void,
msgCallback: (stream: EventSource, evt: MessageEvent) => void,
errorCallback: (stream: EventSource, evt: ErrorEvent) => void,
): EventSource {
const url = hostUpdateUrl(host);
const source = new EventSource(url);
source.addEventListener("open", (e: Event) => {
openCallback(source, e);
});
source.addEventListener("view", (e: Event): void => {
msgCallback(source, e as MessageEvent);
});
source.addEventListener("error", (e: Event) => {
errorCallback(source, e as ErrorEvent);
});
return source;
}

14
frontend/src/component.ts Normal file
View file

@ -0,0 +1,14 @@
import { IPageView } from "./page-view.js";
/** Component that controls an element, rendering output to it. */
export abstract class Component {
protected el: HTMLElement;
constructor(el: HTMLElement) {
this.el = el;
}
public abstract handleView(view: IPageView): void;
public abstract render(): void;
}

2
frontend/src/config.ts Normal file
View file

@ -0,0 +1,2 @@
/** How many seconds a page view is considered "current". */
export const CURRENT_THRESHOLD = 300;

View file

@ -0,0 +1,41 @@
import { IPageView } from "./page-view.js";
import { Table } from "./table.js";
const MAX_VIEWS = 20;
export class CurrentSessions extends Table {
private viewData: IPageView[];
constructor(el: HTMLElement) {
super(el, ["Time", "Session", "Path", "Referrer"]);
this.viewData = [];
}
public handleView(view: IPageView) {
this.viewData.push(view);
this.render();
}
public render() {
this.viewData.sort((a, b) => {
return (b.at.getTime() - a.at.getTime());
});
this.viewData = this.viewData.slice(0, MAX_VIEWS - 1);
super.setData(this.viewData.map((v) => {
let ref: string | HTMLAnchorElement = v.referrer;
if (ref != null) {
ref = document.createElement("a");
ref.href = v.referrer;
ref.innerText = v.referrer;
ref.target = "_blank";
}
return [v.at.toISOString(), v.session, v.path_noq, ref];
}));
super.render();
}
}

51
frontend/src/index.ts Normal file
View file

@ -0,0 +1,51 @@
import { connectListener } from "./archive-client.js";
import { CurrentSessions } from "./current-sessions.js";
import { LiveCounter } from "./live-counter.js";
import { IRemotePageView, parseView } from "./page-view.js";
import { TopPaths } from "./top-paths.js";
import { WorldMap } from "./world-map.js";
let worldMap: WorldMap;
let currentyLiveCounter: LiveCounter;
let currentSessions: CurrentSessions;
let topPaths: TopPaths;
let eventSource: EventSource;
function openCallback(stream: EventSource, e: Event) {
console.log("Opened event stream.");
}
function errorCallback(stream: EventSource, e: ErrorEvent) {
console.log(e);
}
function msgCallback(stream: EventSource, e: MessageEvent) {
const data = JSON.parse(e.data) as IRemotePageView;
console.log("Received data", data);
const parsedData = parseView(data);
currentyLiveCounter.handleView(parsedData);
worldMap.handleView(parsedData);
currentSessions.handleView(parsedData);
topPaths.handleView(parsedData);
}
function main() {
currentyLiveCounter = new LiveCounter(document.getElementById("currently-live-counter")!);
worldMap = new WorldMap(document.getElementById("world-map")!);
currentSessions = new CurrentSessions(document.getElementById("current-sessions-data")!);
topPaths = new TopPaths(document.getElementById("top-paths-data")!);
eventSource = connectListener("localhost:1971", openCallback, msgCallback, errorCallback);
}
if (document.readyState === "interactive") {
main();
} else {
document.addEventListener("readystatechange", () => {
if (document.readyState === "interactive") {
main();
}
});
}

View file

@ -0,0 +1,49 @@
import { Component } from "./component.js";
import { CURRENT_THRESHOLD } from "./config.js";
import { IPageView } from "./page-view.js";
const UPDATE_TIMER = 10;
/**
* LiveCounter shows the amount of users on the site in the last 5 minutes.
*/
export class LiveCounter extends Component {
private sessions: Map<string, Date>;
constructor(el: HTMLElement) {
super(el);
this.sessions = new Map();
setInterval(() => this.removeOld());
}
public handleView(view: IPageView) {
let isLatest = true;
if (this.sessions.has(view.session) && this.sessions.get(view.session)! >= view.at) {
isLatest = false;
}
if (isLatest) {
this.sessions.set(view.session, view.at);
}
this.render();
}
public render() {
this.el.innerText = this.sessions.size.toString();
}
private removeOld() {
const now = Date.now();
for (const [session, at] of this.sessions.entries()) {
if (now - at.getTime() > CURRENT_THRESHOLD * 1000) {
this.sessions.delete(session);
}
}
this.render();
}
}

65
frontend/src/page-view.ts Normal file
View file

@ -0,0 +1,65 @@
export interface IBasePageView {
addr: string;
extra: object;
scrubbed: boolean;
session: string;
path: string;
path_noq: string;
host: string;
referrer: string;
referrer_domain: string;
ua: string;
ua_name: string;
ua_version: string;
os_name: string;
os_version: string;
device_type: string;
screen_w: number;
screen_h: number;
tz_offset: number;
loc_city: string;
loc_country: string;
loc_lat: number;
loc_lon: number;
is_bot: boolean;
}
export interface IRemotePageView extends IBasePageView {
at: string;
}
export interface IPageView extends IBasePageView {
at: Date;
}
/**
* Parse a page view coming from the backend.
*/
export function parseView(view: IRemotePageView): IPageView {
return {
at: new Date(view.at),
addr: view.addr,
extra: view.extra,
scrubbed: view.scrubbed,
session: view.session,
path: view.path,
path_noq: view.path_noq,
host: view.host,
referrer: view.referrer,
referrer_domain: view.referrer_domain,
ua: view.ua,
ua_name: view.ua_name,
ua_version: view.ua_version,
os_name: view.os_name,
os_version: view.os_version,
device_type: view.device_type,
screen_w: view.screen_w,
screen_h: view.screen_h,
tz_offset: view.tz_offset,
loc_city: view.loc_city,
loc_country: view.loc_country,
loc_lat: view.loc_lat,
loc_lon: view.loc_lon,
is_bot: view.is_bot,
};
}

61
frontend/src/table.ts Normal file
View file

@ -0,0 +1,61 @@
import { Component } from "./component.js";
type CellData = string | number | HTMLElement;
type RowData = CellData[];
export abstract class Table extends Component {
private data: RowData[];
private table: HTMLTableElement;
private tbody: HTMLTableSectionElement;
constructor(el: HTMLElement, headings: string[], showFooter: boolean = true) {
super(el);
this.data = [];
this.table = document.createElement("table");
const thead = document.createElement("thead");
const tfoot = document.createElement("tfoot");
for (const heading of headings) {
const th = document.createElement("th");
th.innerText = heading;
thead.append(th);
if (showFooter) { tfoot.append(th.cloneNode(true)); }
}
this.tbody = document.createElement("tbody");
this.table.append(thead);
if (showFooter) { this.table.append(tfoot); }
this.table.append(this.tbody);
this.el.append(this.table);
}
public setData(data: RowData[]) {
this.data = data;
}
public render() {
while (this.tbody.firstChild) {
this.tbody.removeChild(this.tbody.firstChild);
}
for (const row of this.data) {
const tr = document.createElement("tr");
for (const cell of row) {
const td = document.createElement("td");
if (cell instanceof HTMLElement) {
td.append(cell);
} else {
td.innerText = String(cell);
}
tr.append(td);
}
this.tbody.append(tr);
}
}
}

51
frontend/src/top-paths.ts Normal file
View file

@ -0,0 +1,51 @@
import { CURRENT_THRESHOLD } from "./config.js";
import { IPageView } from "./page-view.js";
import { Table } from "./table.js";
const OLD_CLEAR_INTERVAL = 1;
const MAX_LEN = 8;
export class TopPaths extends Table {
private views: IPageView[];
constructor(el: HTMLElement) {
super(el, ["#", "Path"], false);
this.views = [];
setInterval(() => this.clearOld(), OLD_CLEAR_INTERVAL * 1000);
}
public handleView(view: IPageView) {
this.views.push(view);
this.render();
}
public render() {
const hitMap = this.views.reduce((acc, view) => {
let hits = 1;
if (acc.has(view.path_noq)) {
hits += acc.get(view.path_noq)!;
}
acc.set(view.path_noq, hits);
return acc;
}, new Map() as Map<string, number>);
const hitList = Array.from(hitMap.entries());
hitList.sort((a, b) => {
return b[1] - a[1];
});
let transposedList = hitList.map(([path, hits]) => ([hits, path]));
transposedList = transposedList.slice(0, MAX_LEN - 1);
super.setData(transposedList);
super.render();
}
private clearOld() {
const now = Date.now();
this.views = this.views.filter((v) => (now - v.at.getTime()) <= CURRENT_THRESHOLD * 1000);
this.render();
}
}

BIN
frontend/src/world-map.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 171 KiB

62
frontend/src/world-map.ts Normal file
View file

@ -0,0 +1,62 @@
import { Component } from "./component.js";
import { IPageView } from "./page-view.js";
/** How many seconds a spot lives on the map. */
const SPOT_LIFETIME = 60;
/** How often in seconds to update spot opacity. */
const SPOT_UPDATE_INTERVAL = 1;
export class WorldMap extends Component {
private spots: Map<string, [HTMLDivElement, Date]>;
constructor(el: HTMLElement) {
super(el);
this.spots = new Map();
const img = document.createElement("img");
img.alt = "World map";
img.src = "src/world-map.png";
el.append(img);
setInterval(() => this.render(), 1);
}
public handleView(view: IPageView) {
if (view.loc_lat === 0.0 || view.loc_lon === 0.0) {
return;
} else {
let spot: HTMLDivElement;
if (this.spots.has(view.session)) {
spot = this.spots.get(view.session)![0];
} else {
spot = document.createElement("div");
}
spot.className = "spot";
spot.title = (view.loc_city ? `${view.loc_city}, ` : "") + `${view.loc_country} (${view.session})`;
spot.style.bottom = `${(view.loc_lat + 90) / 180 * 100}%`;
spot.style.left = `${(view.loc_lon + 180) / 360 * 100}%`;
this.el.append(spot);
this.spots.set(view.session, [spot, view.at]);
}
}
public render() {
const now = Date.now();
for (const [session, [el, at]] of this.spots.entries()) {
const life = 1 - ((now - at.getTime()) / (SPOT_LIFETIME * 1000));
if (life <= 0) {
this.el.removeChild(el);
this.spots.delete(session);
} else {
el.style.opacity = String(life);
}
}
}
}

101
frontend/style.css Normal file
View file

@ -0,0 +1,101 @@
*, *::before {
box-sizing: border-box;
}
html {
--background-color: #291F1E;
--lighter-bg: #343434;
--heading-color: #799DB4;
--text-color: #E9F0E2;
--alert-color: #F64740;
background-color: var(--background-color);
color: var(--text-color);
font-family: "Helvetica", sans-serif;
font-size: 100%;
width: 100%;
padding: 0;
}
body {
width: 100%;
padding: 5px;
margin: 0;
}
h1, h2, h3, h4, h5, h6 {
font-weight: 100;
color: var(--heading-color);
}
main {
width: 100%;
padding: 10px;
display: grid;
grid-template: 'title wmap'
'clive wmap'
'. wmap'
'csess csess'
/ auto auto;
gap: 10px;
border: 1px solid var(--lighter-bg);
border-radius: 3px;
}
h1 {
font-size: 3rem;
grid-area: title;
margin: 5px 0;
}
a {
color: var(--heading-color);
filter: brightness(150%);
}
table td {
border: 1px solid var(--heading-color);
background-color: var(--lighter-bg);
padding: 5px;
}
#currently-live {
grid-area: clive;
display: flex;
flex-direction: column;
align-items: center;
}
#world-map {
grid-area: wmap;
position: relative;
}
#world-map img {
width: 100%;
}
#world-map .spot {
width: 10px;
height: 10px;
border-radius: 10px;
background-color: var(--alert-color);
opacity: 1;
position: absolute;
}
#current-sessions {
grid-area: csess;
}
#currently-live-counter {
font-size: 5rem;
font-weight: 100;
color: var(--heading-color);
filter: brightness(125%);
}

17
frontend/tsconfig.json Normal file
View file

@ -0,0 +1,17 @@
{
"compileOnSave": true,
"compilerOptions": {
"module": "esnext",
"outDir": "build",
"strict": true,
"moduleResolution": "node",
"sourceMap": true,
"target": "es2019",
},
"include": [
"src/**/*.ts"
],
"exclude": [
"node_modules"
]
}

11
frontend/tslint.json Normal file
View file

@ -0,0 +1,11 @@
{
"defaultSeverity": "error",
"extends": [
"tslint:recommended"
],
"jsRules": {},
"rules": {
"no-console": false
},
"rulesDirectory": []
}