nicd/tilastokeskus
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of gitlab.com:Nicd/tilastokeskus

Browse source
This commit is contained in:
Mikko Ahlroth 2018-06-28 17:14:49 +03:00
commit f0b7e2c504
7 changed files with 155 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -31,6 +31,7 @@ node_modules
# Compiled static files
/priv/static/*
!/priv/static/.gitkeep
!/priv/static/test.html
# Maxmind databases
/priv/maxmind

3
lib/reception/router.ex
View file

@ -1,4 +1,7 @@
defmodule Tilastokeskus.Reception.Router do
@external_resource "priv/static/track.js"
@external_resource "priv/static/test.html"
use Ace.HTTP.Service, port: 1971, cleartext: true
use Raxx.Logger

131
lib/reception/routes/page_view.ex
View file

@ -4,73 +4,87 @@ defmodule Tilastokeskus.Reception.Routes.PageView do
require Logger
@impl Raxx.Server
def handle_request(req, _state) do
at = DateTime.utc_now()
addr = get_addr(req)
ua = parse_ua(req)
{referrer, referrer_noq, referrer_domain} = parse_referrer(req)
def handle_request(req, state) do
body = URI.decode_query(req.body || "")
{path, path_noq, host} = parse_url(body)
screen_w = Map.get(body, "screen_width")
screen_h = Map.get(body, "screen_height")
tz_offset = Map.get(body, "tz_offset")
{path, path_noq, host, authority} = parse_url(body)
{city, country} = get_geoip(addr)
if not check_host(host, state) do
response(400)
|> set_header("content-type", "application/json")
|> set_body(Jason.encode!(%{error: "Host not allowed."}))
else
at = DateTime.utc_now()
# Run in one transaction to avoid multiple DB checkouts
{:ok, response} =
Tilastokeskus.Archive.Repo.transaction(fn ->
session = get_session(req)
addr = get_addr(req)
ua = parse_ua(req)
{referrer, referrer_noq, referrer_domain} = parse_referrer(req)
cset =
PageView.changeset(
%PageView{},
%{
at: at,
session: session,
addr: addr,
path: path,
path_noq: path_noq,
host: host,
referrer: referrer,
referrer_noq: referrer_noq,
referrer_domain: referrer_domain,
ua: unknown_2_str(ua.user_agent),
ua_name: unknown_2_str(ua.client.name),
ua_version: unknown_2_str(ua.client.version),
os_name: unknown_2_str(ua.os.name),
os_version: unknown_2_str(ua.os.version),
device_type: unknown_2_str(ua.device.type),
screen_w: screen_w,
screen_h: screen_h,
tz_offset: tz_offset,
loc_city: city,
loc_country: country
}
)
screen_w = Map.get(body, "screen_width")
screen_h = Map.get(body, "screen_height")
tz_offset = Map.get(body, "tz_offset")
case Tilastokeskus.Archive.Utils.PageView.create(cset) do
{:ok, _} ->
response(200)
|> set_header("content-type", "application/json")
|> set_body(Jason.encode!(%{ok: "OK"}))
|> Raxx.Session.SignedCookie.embed(
session.id,
Tilastokeskus.Reception.Session.config()
{city, country} = get_geoip(addr)
# Run in one transaction to avoid multiple DB checkouts
{:ok, response} =
Tilastokeskus.Archive.Repo.transaction(fn ->
session = get_session(req)
cset =
PageView.changeset(
%PageView{},
%{
at: at,
session: session,
addr: addr,
path: path,
path_noq: path_noq,
host: authority,
referrer: referrer,
referrer_noq: referrer_noq,
referrer_domain: referrer_domain,
ua: unknown_2_str(ua.user_agent),
ua_name: unknown_2_str(ua.client.name),
ua_version: unknown_2_str(ua.client.version),
os_name: unknown_2_str(ua.os.name),
os_version: unknown_2_str(ua.os.version),
device_type: unknown_2_str(ua.device.type),
screen_w: screen_w,
screen_h: screen_h,
tz_offset: tz_offset,
loc_city: city,
loc_country: country
}
)
{:error, err} ->
Logger.error("Error saving pageview: #{inspect(err)}")
case Tilastokeskus.Archive.Utils.PageView.create(cset) do
{:ok, _} ->
response(200)
|> set_header("content-type", "application/json")
|> set_body(Jason.encode!(%{ok: "OK"}))
|> Raxx.Session.SignedCookie.embed(
session.id,
Tilastokeskus.Reception.Session.config()
)
response(500)
|> set_header("content-type", "application/json")
|> set_body(Jason.encode!(%{error: "ERROR"}))
end
end)
{:error, err} ->
Logger.error("Error saving pageview: #{inspect(err)}")
response
response(500)
|> set_header("content-type", "application/json")
|> set_body(Jason.encode!(%{error: "ERROR"}))
end
end)
response
end
end
defp check_host(host, state) do
allowed_hosts = Keyword.get(state, :hosts, [])
# If no hosts are specified, everything goes
Enum.empty?(allowed_hosts) || Enum.member?(allowed_hosts, host)
end
defp get_addr(req) do
@ -166,6 +180,7 @@ defmodule Tilastokeskus.Reception.Routes.PageView do
{
if(not is_nil(parsed.query), do: "#{parsed.path}?#{parsed.query}", else: parsed.path),
parsed.path,
parsed.host,
parsed.authority
}
end

10
lib/tilastokeskus/application.ex
View file

@ -9,11 +9,12 @@ defmodule Tilastokeskus.Application do
:ok = Tilastokeskus.Reception.Session.validate_config()
port = (System.get_env("PORT") || "1971") |> String.to_integer()
hosts = get_hosts()
# List all child processes to be supervised
children = [
{Tilastokeskus.Archive.Repo, []},
{Tilastokeskus.Reception.Router, [port: port]}
{Tilastokeskus.Reception.Router, [[hosts: hosts], [port: port]]}
]
# See https://hexdocs.pm/elixir/Supervisor.html
@ -21,4 +22,11 @@ defmodule Tilastokeskus.Application do
opts = [strategy: :one_for_one, name: Tilastokeskus.Supervisor]
Supervisor.start_link(children, opts)
end
defp get_hosts() do
case System.get_env("TILASTOKESKUS_HOSTS") do
nil -> ""
hosts -> String.split(hosts, ",")
end
end
end

2
package.json
View file

@ -27,4 +27,4 @@
"devDependencies": {
"uglify-es": "^3.3.9"
}
}
}

16
priv/static/test.html Normal file
View file

@ -0,0 +1,16 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>Tilastokeskus tester page</title>
</head>
<body>
<script type="text/javascript">
window.tilastokeskus_url = '/track';
</script>
<script type="text/javascript" src="/track.js"></script>
</body>
</html>

65
priv/ua/device.mobiles.yml
View file

@ -54,7 +54,7 @@ SFR:
# HTC
HTC:
regex: 'HTC|Sprint (?:APA|ATP)|ADR(?!910L)[a-z0-9]+|NexusHD2|Amaze[ _]4G[);/ ]|(Desire|Sensation|Evo ?3D|IncredibleS|Wildfire|Butterfly)[ _]?([^;/]+) Build|(Amaze[ _]4G|One ?[XELSV\+]+)[);/ ]|SPV E6[05]0|One M8|X525a|PG86100|PC36100|XV6975|PJ83100[);/ ]'
regex: 'HTC|Sprint (?:APA|ATP)|ADR(?!910L)[a-z0-9]+|NexusHD2|Amaze[ _]4G[);/ ]|(Desire|Sensation|Evo ?3D|IncredibleS|Wildfire|Butterfly)[ _]?([^;/]+) Build|(Amaze[ _]4G|One ?[XELSV\+]+)[);/ ]|SPV E6[05]0|One M8|X525a|PG86100|PC36100|XV6975|PJ83100[);/ ]|2PYB2'
device: 'smartphone'
models:
# explicit smartphone models
@ -78,6 +78,8 @@ HTC:
model: 'Vox'
- regex: 'X525a'
model: 'One X+'
- regex: '2PYB2'
model: 'Bolt'
- regex: 'NexusHD2' # custom rom for hd2
model: 'HD2'
@ -1077,10 +1079,12 @@ Vivo:
regex: '(?:VIV-|BBG-)?vivo'
device: 'smartphone'
models:
- regex: '(?:VIV-|BBG-)?vivo[ _]1601 Build'
model: 'V5'
- regex: '(?:VIV-|BBG-)?vivo[ _]([^/;]+) Build'
model: 'Vivo $1'
model: '$1'
- regex: '(?:VIV-|BBG-)?vivo[ _]([^);/]+)[);/]+'
model: 'Vivo $1'
model: '$1'
# Bird
Bird:
@ -2643,7 +2647,7 @@ Huawei:
model: 'Nova 2I'
- regex: 'EVA-L[012]9'
model: 'P9'
- regex: 'VNS-L21'
- regex: 'VNS-L[23]1'
model: 'P9 Lite'
- regex: 'VTR-L09'
model: 'P10'
@ -2651,6 +2655,8 @@ Huawei:
model: 'P10 Lite'
- regex: 'BAC-L[02]3'
model: 'P10 Selfie'
- regex: 'VKY-(?:L09|L029|AL00)'
model: 'P10 Plus'
- regex: 'ANE-LX3'
model: 'P20 Lite'
- regex: 'CLT-L[02]9'
@ -3278,6 +3284,10 @@ Lenovo:
model: 'K6 Note'
- regex: 'K33a42 Build'
model: 'K6 Power'
- regex: 'P1ma40 Build'
model: 'P1 Vibe'
- regex: 'P2a42 Build'
model: 'P2'
- regex: 'Lenovo ([^/;]*) Build'
model: '$1'
- regex: '(?:LNV-|Lenovo-)?Lenovo[ \-_]?([a-z0-9_+\-]+)'
@ -3405,8 +3415,18 @@ LG:
model: 'PRADA phone L-02D'
- regex: 'L-07C'
model: 'Optimus Bright L-07C'
- regex: '(?:LG-)?F800(?:S|K|L)?'
model: 'V20'
- regex: '(?:LG-)?H91[058]'
model: 'V20'
- regex: '(?:LG-)?H990(?:DS|N|T)?'
model: 'V20'
- regex: '(?:LG-)?LS997'
model: 'V20'
- regex: '(?:LG-)?VS995'
model: 'V20'
- regex: '(?:LG-)?US999'
model: 'V20'
- regex: '(?:LG-)?VS410PP'
model: 'Optimus Zone'
- regex: '(?:LG-)?VS415PP'
@ -3415,14 +3435,18 @@ LG:
model: 'Optimus Zone 3'
- regex: '(?:LG-)?K100'
model: 'K3'
- regex: '(?:LG-?)?MS330'
- regex: '(?:LG-?)?(?:K|MS)330'
model: 'K7'
- regex: '(?:LG-)?K430(?:N|DS|DSF|DSY)?'
- regex: '(?:LG-)?K420(?:N)?'
model: 'K10'
- regex: '(?:LG-)?K430(?:DS|DSF|DSY)?'
model: 'K10'
- regex: '(?:LG-)?M250N?'
model: 'K10 (2017)'
- regex: '(?:LG-)?F670[KLS]?'
model: 'K10'
- regex: '(?:LG-)?M700(?:N|A|DSK|AN)?'
model: 'Q6'
- regex: '(?:LG-)?K371'
model: 'Phoenix 2'
- regex: '(?:LG-)?M150'
@ -4370,7 +4394,7 @@ OnePlus:
# oppo
OPPO:
regex: '(?:OB-)?OPPO[ _]?([a-z0-9]+)|N1T|(?:X90[07][0679]|U707T?|X909T?|R(?:10[01]1|2001|201[07]|6007|7005|7007|80[13579]|81[13579]|82[01379]|83[013]|800[067]|8015|810[679]|811[13]|820[057])[KLSTW]?)|N520[79]|N5117|A37f[);/ ]|A1601|CPH[0-9]{4}'
regex: '(?:OB-)?OPPO[ _]?([a-z0-9]+)|N1T|(?:X90[07][0679]|U707T?|X909T?|R(?:10[01]1|2001|201[07]|6007|7005|7007|80[13579]|81[13579]|82[01379]|83[013]|800[067]|8015|810[679]|811[13]|820[057])[KLSTW]?)|N520[79]|N5117|A37fw?[);/ ]|A1601|CPH[0-9]{4}'
device: 'smartphone'
models:
- regex: '(?:OPPO[ _]?)?U707T?[);/ ]'
@ -4435,8 +4459,8 @@ OPPO:
model: 'F3 Plus'
- regex: '(?:OPPO[ _]?)?CPH1701[);/ ]'
model: 'A57'
- regex: '(?:OPPO[ _]?)?A37f Build'
model: 'A37f'
- regex: '(?:OPPO[ _]?)?A37f(w)? Build'
model: 'A37f$1'
- regex: '(?:OPPO[ _]?)?A1601 Build'
model: 'F1s'
- regex: '(?:OPPO[ _]?)?R1011 Build'
@ -4837,6 +4861,9 @@ Samsung:
- regex: '(?:SAMSUNG-)?SM-T230(?:NU)?'
device: 'tablet'
model: 'GALAXY Tab 4 7.0" WiFi'
- regex: '(?:SAMSUNG-)?SM-T231'
device: 'tablet'
model: 'GALAXY Tab 4 7.0" 3G'
- regex: '(?:SAMSUNG-)?SM-T310'
device: 'tablet'
model: 'GALAXY Tab 3 8.0" WiFi'
@ -5062,7 +5089,7 @@ Samsung:
model: 'GALAXY S5 LTE+'
- regex: '(?:SAMSUNG-)?SM-G903F'
model: 'GALAXY S5 Neo'
- regex: '(?:SAMSUNG-)?SM-G920[FI]'
- regex: '(?:SAMSUNG-)?SM-G920[FIP]'
model: 'GALAXY S6'
- regex: '(?:SAMSUNG-)?SM-G925[FI]'
model: 'GALAXY S6 edge'
@ -5076,6 +5103,10 @@ Samsung:
model: 'GALAXY S8'
- regex: '(?:SAMSUNG-)?SM-G955[FU0]'
model: 'GALAXY S8+'
- regex: '(?:SAMSUNG-)?SM-G960[FUW0]'
model: 'GALAXY S9'
- regex: '(?:SAMSUNG-)?SM-G965[FUW0]'
model: 'GALAXY S9+'
- regex: '(?:SAMSUNG-)?SCH-I200'
model: 'GALAXY Stellar'
- regex: '(?:SAMSUNG-)?SCH-I829'
@ -5118,6 +5149,8 @@ Samsung:
model: 'GALAXY J5 (2015)'
- regex: '(?:SAMSUNG-)?SM-J510[FM]N'
model: 'GALAXY J5 (2016)'
- regex: '(?:SAMSUNG-)?SM-J530[FY]'
model: 'GALAXY J5 (2017)'
- regex: '(?:SAMSUNG-)?SM-G570F'
model: 'GALAXY J5 Prime'
- regex: '(?:SAMSUNG-)?SM-J7[01]0(?:F|FN|H|M|MN)'
@ -5126,10 +5159,16 @@ Samsung:
model: 'GALAXY J7+ (C7)'
- regex: '(?:SAMSUNG-)?SM-G610[FMY]'
model: 'GALAXY J7 Prime'
- regex: '(?:SAMSUNG-)?SM-G615[F]'
model: 'GALAXY J7 Max'
- regex: '(?:SAMSUNG-)?SM-J730(?:F|G|GM)'
model: 'GALAXY J7 Pro'
- regex: '(?:SAMSUNG-)?SM-J701(?:F|M)'
model: 'GALAXY J7 Core' # also GALAXY J7 Nxt and GALAXY J7 Neo
- regex: '(?:SAMSUNG-)?SM-G850F'
model: 'GALAXY Alpha'
- regex: '(?:SAMSUNG-)?SM-G550FY'
model: 'GALAXY On5'
- regex: '(?:SAMSUNG-)?SM-G600FY'
model: 'GALAXY On7'
- regex: '(?:SAMSUNG-)?SM-C900[FY0]'
@ -5160,19 +5199,19 @@ Samsung:
- regex: '(?:SAMSUNG-)?GT-N7105'
model: 'GALAXY Note II LTE'
device: 'phablet'
- regex: '(?:SAMSUNG-)?SM-G7105'
- regex: '(?:SAMSUNG-)?SM-G710[25]'
model: 'GALAXY Grand 2'
device: 'phablet'
- regex: '(?:SAMSUNG-)?SM-G53(?:0F|0FZ|0Y|0H|0FZ|1F|1H)'
model: 'GALAXY Grand Prime'
device: 'phablet'
- regex: '(?:SAMSUNG-)?SM-G532F'
model: 'GALAXY Grand Prime Plus'
model: 'GALAXY Grand Prime Plus' # also GALAXY J2 Prime
device: 'phablet'
- regex: '(?:SAMSUNG-)?SM-G532MT'
model: 'GALAXY J2 Prime (TV)'
device: 'phablet'
- regex: '(?:SAMSUNG-)?SM-G532M'
- regex: '(?:SAMSUNG-)?SM-G532[MG]'
model: 'GALAXY J2 Prime'
device: 'phablet'
- regex: '(?:SAMSUNG-)?SM-N7502'