From 774ff98706d1610e0ea7616548294cd9e73744f2 Mon Sep 17 00:00:00 2001 From: Mikko Ahlroth Date: Thu, 28 Jun 2018 11:50:11 +0300 Subject: [PATCH 1/4] Remove unused variable --- lib/reception/routes/page_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/reception/routes/page_view.ex b/lib/reception/routes/page_view.ex index a197ea7..21fac89 100644 --- a/lib/reception/routes/page_view.ex +++ b/lib/reception/routes/page_view.ex @@ -172,7 +172,7 @@ defmodule Tilastokeskus.Reception.Routes.PageView do end defp get_geoip(addr) do - %{city: city, country: country} = res = Geolix.lookup(addr, as: :raw) + %{city: city, country: country} = Geolix.lookup(addr, as: :raw) { get_in(city, [:city, :names, :en]), From 768939f00f66b02a463bd7c4507a5fafc3c9f68f Mon Sep 17 00:00:00 2001 From: Mikko Ahlroth Date: Thu, 28 Jun 2018 12:35:16 +0300 Subject: [PATCH 2/4] Add support for specifying acceptable hosts --- lib/reception/routes/page_view.ex | 131 +++++++++++++++++------------- lib/tilastokeskus/application.ex | 10 ++- 2 files changed, 82 insertions(+), 59 deletions(-) diff --git a/lib/reception/routes/page_view.ex b/lib/reception/routes/page_view.ex index 21fac89..17cb5d2 100644 --- a/lib/reception/routes/page_view.ex +++ b/lib/reception/routes/page_view.ex @@ -4,73 +4,87 @@ defmodule Tilastokeskus.Reception.Routes.PageView do require Logger @impl Raxx.Server - def handle_request(req, _state) do - at = DateTime.utc_now() - - addr = get_addr(req) - ua = parse_ua(req) - {referrer, referrer_noq, referrer_domain} = parse_referrer(req) - + def handle_request(req, state) do body = URI.decode_query(req.body || "") - {path, path_noq, host} = parse_url(body) - screen_w = Map.get(body, "screen_width") - screen_h = Map.get(body, "screen_height") - tz_offset = Map.get(body, "tz_offset") + {path, path_noq, host, authority} = parse_url(body) - {city, country} = get_geoip(addr) + if not check_host(host, state) do + response(400) + |> set_header("content-type", "application/json") + |> set_body(Jason.encode!(%{error: "Host not allowed."})) + else + at = DateTime.utc_now() - # Run in one transaction to avoid multiple DB checkouts - {:ok, response} = - Tilastokeskus.Archive.Repo.transaction(fn -> - session = get_session(req) + addr = get_addr(req) + ua = parse_ua(req) + {referrer, referrer_noq, referrer_domain} = parse_referrer(req) - cset = - PageView.changeset( - %PageView{}, - %{ - at: at, - session: session, - addr: addr, - path: path, - path_noq: path_noq, - host: host, - referrer: referrer, - referrer_noq: referrer_noq, - referrer_domain: referrer_domain, - ua: unknown_2_str(ua.user_agent), - ua_name: unknown_2_str(ua.client.name), - ua_version: unknown_2_str(ua.client.version), - os_name: unknown_2_str(ua.os.name), - os_version: unknown_2_str(ua.os.version), - device_type: unknown_2_str(ua.device.type), - screen_w: screen_w, - screen_h: screen_h, - tz_offset: tz_offset, - loc_city: city, - loc_country: country - } - ) + screen_w = Map.get(body, "screen_width") + screen_h = Map.get(body, "screen_height") + tz_offset = Map.get(body, "tz_offset") - case Tilastokeskus.Archive.Utils.PageView.create(cset) do - {:ok, _} -> - response(200) - |> set_header("content-type", "application/json") - |> set_body(Jason.encode!(%{ok: "OK"})) - |> Raxx.Session.SignedCookie.embed( - session.id, - Tilastokeskus.Reception.Session.config() + {city, country} = get_geoip(addr) + + # Run in one transaction to avoid multiple DB checkouts + {:ok, response} = + Tilastokeskus.Archive.Repo.transaction(fn -> + session = get_session(req) + + cset = + PageView.changeset( + %PageView{}, + %{ + at: at, + session: session, + addr: addr, + path: path, + path_noq: path_noq, + host: authority, + referrer: referrer, + referrer_noq: referrer_noq, + referrer_domain: referrer_domain, + ua: unknown_2_str(ua.user_agent), + ua_name: unknown_2_str(ua.client.name), + ua_version: unknown_2_str(ua.client.version), + os_name: unknown_2_str(ua.os.name), + os_version: unknown_2_str(ua.os.version), + device_type: unknown_2_str(ua.device.type), + screen_w: screen_w, + screen_h: screen_h, + tz_offset: tz_offset, + loc_city: city, + loc_country: country + } ) - {:error, err} -> - Logger.error("Error saving pageview: #{inspect(err)}") + case Tilastokeskus.Archive.Utils.PageView.create(cset) do + {:ok, _} -> + response(200) + |> set_header("content-type", "application/json") + |> set_body(Jason.encode!(%{ok: "OK"})) + |> Raxx.Session.SignedCookie.embed( + session.id, + Tilastokeskus.Reception.Session.config() + ) - response(500) - |> set_header("content-type", "application/json") - |> set_body(Jason.encode!(%{error: "ERROR"})) - end - end) + {:error, err} -> + Logger.error("Error saving pageview: #{inspect(err)}") - response + response(500) + |> set_header("content-type", "application/json") + |> set_body(Jason.encode!(%{error: "ERROR"})) + end + end) + + response + end + end + + defp check_host(host, state) do + allowed_hosts = Keyword.get(state, :hosts, []) + + # If no hosts are specified, everything goes + Enum.empty?(allowed_hosts) || Enum.member?(allowed_hosts, host) end defp get_addr(req) do @@ -166,6 +180,7 @@ defmodule Tilastokeskus.Reception.Routes.PageView do { if(not is_nil(parsed.query), do: "#{parsed.path}?#{parsed.query}", else: parsed.path), parsed.path, + parsed.host, parsed.authority } end diff --git a/lib/tilastokeskus/application.ex b/lib/tilastokeskus/application.ex index 829f211..da87bdf 100644 --- a/lib/tilastokeskus/application.ex +++ b/lib/tilastokeskus/application.ex @@ -9,11 +9,12 @@ defmodule Tilastokeskus.Application do :ok = Tilastokeskus.Reception.Session.validate_config() port = (System.get_env("PORT") || "1971") |> String.to_integer() + hosts = get_hosts() # List all child processes to be supervised children = [ {Tilastokeskus.Archive.Repo, []}, - {Tilastokeskus.Reception.Router, [port: port]} + {Tilastokeskus.Reception.Router, [[hosts: hosts], [port: port]]} ] # See https://hexdocs.pm/elixir/Supervisor.html @@ -21,4 +22,11 @@ defmodule Tilastokeskus.Application do opts = [strategy: :one_for_one, name: Tilastokeskus.Supervisor] Supervisor.start_link(children, opts) end + + defp get_hosts() do + case System.get_env("TILASTOKESKUS_HOSTS") do + nil -> "" + hosts -> String.split(hosts, ",") + end + end end From 86c9ed27ab2db5dd6460d675a698da87622c61a1 Mon Sep 17 00:00:00 2001 From: Mikko Ahlroth Date: Thu, 28 Jun 2018 12:35:38 +0300 Subject: [PATCH 3/4] Add tester page for localhost development --- .gitignore | 1 + lib/reception/router.ex | 3 +++ package.json | 2 +- priv/static/test.html | 16 ++++++++++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 priv/static/test.html diff --git a/.gitignore b/.gitignore index d4e696f..535db73 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ node_modules # Compiled static files /priv/static/* !/priv/static/.gitkeep +!/priv/static/test.html # Maxmind databases /priv/maxmind diff --git a/lib/reception/router.ex b/lib/reception/router.ex index d770322..0c02b3a 100644 --- a/lib/reception/router.ex +++ b/lib/reception/router.ex @@ -1,4 +1,7 @@ defmodule Tilastokeskus.Reception.Router do + @external_resource "priv/static/track.js" + @external_resource "priv/static/test.html" + use Ace.HTTP.Service, port: 1971, cleartext: true use Raxx.Logger diff --git a/package.json b/package.json index d79484d..356fec6 100644 --- a/package.json +++ b/package.json @@ -27,4 +27,4 @@ "devDependencies": { "uglify-es": "^3.3.9" } -} \ No newline at end of file +} diff --git a/priv/static/test.html b/priv/static/test.html new file mode 100644 index 0000000..2b3de37 --- /dev/null +++ b/priv/static/test.html @@ -0,0 +1,16 @@ + + + + + + Tilastokeskus tester page + + + + + + + + \ No newline at end of file From 2f06d203efbaf7b95a587fae69dba8d3c2aa949a Mon Sep 17 00:00:00 2001 From: Mikko Ahlroth Date: Thu, 28 Jun 2018 12:36:19 +0300 Subject: [PATCH 4/4] Update UA database --- priv/ua/device.mobiles.yml | 65 ++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/priv/ua/device.mobiles.yml b/priv/ua/device.mobiles.yml index 322dc53..c337e25 100644 --- a/priv/ua/device.mobiles.yml +++ b/priv/ua/device.mobiles.yml @@ -54,7 +54,7 @@ SFR: # HTC HTC: - regex: 'HTC|Sprint (?:APA|ATP)|ADR(?!910L)[a-z0-9]+|NexusHD2|Amaze[ _]4G[);/ ]|(Desire|Sensation|Evo ?3D|IncredibleS|Wildfire|Butterfly)[ _]?([^;/]+) Build|(Amaze[ _]4G|One ?[XELSV\+]+)[);/ ]|SPV E6[05]0|One M8|X525a|PG86100|PC36100|XV6975|PJ83100[);/ ]' + regex: 'HTC|Sprint (?:APA|ATP)|ADR(?!910L)[a-z0-9]+|NexusHD2|Amaze[ _]4G[);/ ]|(Desire|Sensation|Evo ?3D|IncredibleS|Wildfire|Butterfly)[ _]?([^;/]+) Build|(Amaze[ _]4G|One ?[XELSV\+]+)[);/ ]|SPV E6[05]0|One M8|X525a|PG86100|PC36100|XV6975|PJ83100[);/ ]|2PYB2' device: 'smartphone' models: # explicit smartphone models @@ -78,6 +78,8 @@ HTC: model: 'Vox' - regex: 'X525a' model: 'One X+' + - regex: '2PYB2' + model: 'Bolt' - regex: 'NexusHD2' # custom rom for hd2 model: 'HD2' @@ -1077,10 +1079,12 @@ Vivo: regex: '(?:VIV-|BBG-)?vivo' device: 'smartphone' models: + - regex: '(?:VIV-|BBG-)?vivo[ _]1601 Build' + model: 'V5' - regex: '(?:VIV-|BBG-)?vivo[ _]([^/;]+) Build' - model: 'Vivo $1' + model: '$1' - regex: '(?:VIV-|BBG-)?vivo[ _]([^);/]+)[);/]+' - model: 'Vivo $1' + model: '$1' # Bird Bird: @@ -2643,7 +2647,7 @@ Huawei: model: 'Nova 2I' - regex: 'EVA-L[012]9' model: 'P9' - - regex: 'VNS-L21' + - regex: 'VNS-L[23]1' model: 'P9 Lite' - regex: 'VTR-L09' model: 'P10' @@ -2651,6 +2655,8 @@ Huawei: model: 'P10 Lite' - regex: 'BAC-L[02]3' model: 'P10 Selfie' + - regex: 'VKY-(?:L09|L029|AL00)' + model: 'P10 Plus' - regex: 'ANE-LX3' model: 'P20 Lite' - regex: 'CLT-L[02]9' @@ -3278,6 +3284,10 @@ Lenovo: model: 'K6 Note' - regex: 'K33a42 Build' model: 'K6 Power' + - regex: 'P1ma40 Build' + model: 'P1 Vibe' + - regex: 'P2a42 Build' + model: 'P2' - regex: 'Lenovo ([^/;]*) Build' model: '$1' - regex: '(?:LNV-|Lenovo-)?Lenovo[ \-_]?([a-z0-9_+\-]+)' @@ -3405,8 +3415,18 @@ LG: model: 'PRADA phone L-02D' - regex: 'L-07C' model: 'Optimus Bright L-07C' + - regex: '(?:LG-)?F800(?:S|K|L)?' + model: 'V20' + - regex: '(?:LG-)?H91[058]' + model: 'V20' + - regex: '(?:LG-)?H990(?:DS|N|T)?' + model: 'V20' + - regex: '(?:LG-)?LS997' + model: 'V20' - regex: '(?:LG-)?VS995' model: 'V20' + - regex: '(?:LG-)?US999' + model: 'V20' - regex: '(?:LG-)?VS410PP' model: 'Optimus Zone' - regex: '(?:LG-)?VS415PP' @@ -3415,14 +3435,18 @@ LG: model: 'Optimus Zone 3' - regex: '(?:LG-)?K100' model: 'K3' - - regex: '(?:LG-?)?MS330' + - regex: '(?:LG-?)?(?:K|MS)330' model: 'K7' - - regex: '(?:LG-)?K430(?:N|DS|DSF|DSY)?' + - regex: '(?:LG-)?K420(?:N)?' + model: 'K10' + - regex: '(?:LG-)?K430(?:DS|DSF|DSY)?' model: 'K10' - regex: '(?:LG-)?M250N?' model: 'K10 (2017)' - regex: '(?:LG-)?F670[KLS]?' model: 'K10' + - regex: '(?:LG-)?M700(?:N|A|DSK|AN)?' + model: 'Q6' - regex: '(?:LG-)?K371' model: 'Phoenix 2' - regex: '(?:LG-)?M150' @@ -4370,7 +4394,7 @@ OnePlus: # oppo OPPO: - regex: '(?:OB-)?OPPO[ _]?([a-z0-9]+)|N1T|(?:X90[07][0679]|U707T?|X909T?|R(?:10[01]1|2001|201[07]|6007|7005|7007|80[13579]|81[13579]|82[01379]|83[013]|800[067]|8015|810[679]|811[13]|820[057])[KLSTW]?)|N520[79]|N5117|A37f[);/ ]|A1601|CPH[0-9]{4}' + regex: '(?:OB-)?OPPO[ _]?([a-z0-9]+)|N1T|(?:X90[07][0679]|U707T?|X909T?|R(?:10[01]1|2001|201[07]|6007|7005|7007|80[13579]|81[13579]|82[01379]|83[013]|800[067]|8015|810[679]|811[13]|820[057])[KLSTW]?)|N520[79]|N5117|A37fw?[);/ ]|A1601|CPH[0-9]{4}' device: 'smartphone' models: - regex: '(?:OPPO[ _]?)?U707T?[);/ ]' @@ -4435,8 +4459,8 @@ OPPO: model: 'F3 Plus' - regex: '(?:OPPO[ _]?)?CPH1701[);/ ]' model: 'A57' - - regex: '(?:OPPO[ _]?)?A37f Build' - model: 'A37f' + - regex: '(?:OPPO[ _]?)?A37f(w)? Build' + model: 'A37f$1' - regex: '(?:OPPO[ _]?)?A1601 Build' model: 'F1s' - regex: '(?:OPPO[ _]?)?R1011 Build' @@ -4837,6 +4861,9 @@ Samsung: - regex: '(?:SAMSUNG-)?SM-T230(?:NU)?' device: 'tablet' model: 'GALAXY Tab 4 7.0" WiFi' + - regex: '(?:SAMSUNG-)?SM-T231' + device: 'tablet' + model: 'GALAXY Tab 4 7.0" 3G' - regex: '(?:SAMSUNG-)?SM-T310' device: 'tablet' model: 'GALAXY Tab 3 8.0" WiFi' @@ -5062,7 +5089,7 @@ Samsung: model: 'GALAXY S5 LTE+' - regex: '(?:SAMSUNG-)?SM-G903F' model: 'GALAXY S5 Neo' - - regex: '(?:SAMSUNG-)?SM-G920[FI]' + - regex: '(?:SAMSUNG-)?SM-G920[FIP]' model: 'GALAXY S6' - regex: '(?:SAMSUNG-)?SM-G925[FI]' model: 'GALAXY S6 edge' @@ -5076,6 +5103,10 @@ Samsung: model: 'GALAXY S8' - regex: '(?:SAMSUNG-)?SM-G955[FU0]' model: 'GALAXY S8+' + - regex: '(?:SAMSUNG-)?SM-G960[FUW0]' + model: 'GALAXY S9' + - regex: '(?:SAMSUNG-)?SM-G965[FUW0]' + model: 'GALAXY S9+' - regex: '(?:SAMSUNG-)?SCH-I200' model: 'GALAXY Stellar' - regex: '(?:SAMSUNG-)?SCH-I829' @@ -5118,6 +5149,8 @@ Samsung: model: 'GALAXY J5 (2015)' - regex: '(?:SAMSUNG-)?SM-J510[FM]N' model: 'GALAXY J5 (2016)' + - regex: '(?:SAMSUNG-)?SM-J530[FY]' + model: 'GALAXY J5 (2017)' - regex: '(?:SAMSUNG-)?SM-G570F' model: 'GALAXY J5 Prime' - regex: '(?:SAMSUNG-)?SM-J7[01]0(?:F|FN|H|M|MN)' @@ -5126,10 +5159,16 @@ Samsung: model: 'GALAXY J7+ (C7)' - regex: '(?:SAMSUNG-)?SM-G610[FMY]' model: 'GALAXY J7 Prime' + - regex: '(?:SAMSUNG-)?SM-G615[F]' + model: 'GALAXY J7 Max' - regex: '(?:SAMSUNG-)?SM-J730(?:F|G|GM)' model: 'GALAXY J7 Pro' + - regex: '(?:SAMSUNG-)?SM-J701(?:F|M)' + model: 'GALAXY J7 Core' # also GALAXY J7 Nxt and GALAXY J7 Neo - regex: '(?:SAMSUNG-)?SM-G850F' model: 'GALAXY Alpha' + - regex: '(?:SAMSUNG-)?SM-G550FY' + model: 'GALAXY On5' - regex: '(?:SAMSUNG-)?SM-G600FY' model: 'GALAXY On7' - regex: '(?:SAMSUNG-)?SM-C900[FY0]' @@ -5160,19 +5199,19 @@ Samsung: - regex: '(?:SAMSUNG-)?GT-N7105' model: 'GALAXY Note II LTE' device: 'phablet' - - regex: '(?:SAMSUNG-)?SM-G7105' + - regex: '(?:SAMSUNG-)?SM-G710[25]' model: 'GALAXY Grand 2' device: 'phablet' - regex: '(?:SAMSUNG-)?SM-G53(?:0F|0FZ|0Y|0H|0FZ|1F|1H)' model: 'GALAXY Grand Prime' device: 'phablet' - regex: '(?:SAMSUNG-)?SM-G532F' - model: 'GALAXY Grand Prime Plus' + model: 'GALAXY Grand Prime Plus' # also GALAXY J2 Prime device: 'phablet' - regex: '(?:SAMSUNG-)?SM-G532MT' model: 'GALAXY J2 Prime (TV)' device: 'phablet' - - regex: '(?:SAMSUNG-)?SM-G532M' + - regex: '(?:SAMSUNG-)?SM-G532[MG]' model: 'GALAXY J2 Prime' device: 'phablet' - regex: '(?:SAMSUNG-)?SM-N7502'