diff -urPN 0.1.0/Makefile 1.0.0/Makefile --- 0.1.0/Makefile Mon Aug 10 21:18:03 2020 +++ 1.0.0/Makefile Fri Mar 18 15:52:41 2022 @@ -1,3 +1,4 @@ +VERSION=1.0.0 PREFIX = /usr/local MANPREFIX = $(PREFIX)/share/man @@ -22,10 +23,8 @@ install: mkdir -p $(DESTDIR)$(PREFIX)/bin/ - mkdir -p $(DESTDIR)$(PREFIX)/share/fl mkdir -p $(DESTDIR)$(MANPREFIX)/man1/ cp -f $(BIN) $(DESTDIR)$(PREFIX)/bin/ - cp -r share/*.awk $(DESTDIR)$(PREFIX)/share/fl cp -f $(MAN) $(DESTDIR)$(MANPREFIX)/man1/ for bin in $(BIN); do \ chmod 755 $(DESTIR)$(PREFIX)/bin/$$bin; \ @@ -36,10 +35,17 @@ uninstall: - rm -rf $(DESTDIR$(PREFIX)/share/fl for bin in $(BIN); do \ rm -f $(DESTDIR)$(PREFIX)/bin/$$bin; \ done for man in $(MAN); do \ rm -f $(MANPREFIX)/man1/$$man; \ done + +dist: + mkdir -p fl@$(VERSION)/fl + cp $(BIN) fl@$(VERSION)/fl + cp $(MAN) fl@$(VERSION)/fl + cp Makefile README NEWS LICENSE fl@$(VERSION)/fl + tar -C fl@$(VERSION) -czvf fl@$(VERSION).tgz fl + rm -rf fl@$(VERSION) diff -urPN 0.1.0/NEWS 1.0.0/NEWS --- 0.1.0/NEWS Thu Jan 1 01:00:00 1970 +++ 1.0.0/NEWS Fri Mar 18 15:52:41 2022 @@ -0,0 +1,34 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.0.0] - 2022-03-04 + +### Added + +- Add `NEWS` file +- Add `dist` rule to `Makefile` +- Add missing `flvu.1` `man` file to dist package + +### Changed + +- Hardcode all log fields in scripts +- Write simple description in README rather than `man` contents +- Update `man` pages to reflect new path patterns + +### Removed + +- Delete `share` directory and references +- Remove dependency on [`cl`](https://adi.onl/cl.html) and [`cbl`](https://adi.onl/cbl.html) + +## [0.0.1] + +### Added + +- Forwarded Log analytics scripts + +[1.0.0]: https://adi.onl/fl/0.1.0..1.0.0 +[0.1.0]: https://adi.onl/fl/0.1.0 diff -urPN 0.1.0/README 1.0.0/README --- 0.1.0/README Mon Aug 10 21:18:03 2020 +++ 1.0.0/README Fri Mar 18 15:52:41 2022 @@ -1,516 +1,80 @@ # fl -forwarded log +Analytics for `HTTP` logs in Forwarded Log Format -## flv(1) +## Install -### NAME +On a Linux or BSD machine, in a terminal, run: -**flv** - forwarded log views + mkdir -p ~/src + cd ~/src + wget -qO - https://adi.onl/fl/fl@1.0.0.tgz | tar -xzvf - + cd fl + make install PREFIX=$HOME -### SYNOPSIS +## Uninstall -**flv** -\[*file ...*] + cd ~/src/fl + make uninstall PREFIX=$HOME -### DESCRIPTION +## Usage -The -**flv** -utility reads -each -\[*file ...*] -in -*Forwarded Log Format* -and prints only the -*views* -to standard output. +The package provides 8 scripts `flp`, `flu`, `flv`, `flr` and +`flvp`, `flvu` , `flvv` and `flvr`. -*views* -are defined as -*GET* -requests -to paths ending in -*/* -or -*.html* -that returned a -*200* -HTTP response status code. +The **fl** prefix stands for _Forwarded Log_ and the suffixes stand for +_**p**ages_, _**u**nique_, _**v**iews_ and _**r**eferrers_. -If -\[*file ...*] -is omitted, -**flv** -reads from standard input. +The middle **v** in the second set stands for _virtual host_. The second +set of 4 scripts is for the Forwarded Log with virtual hosts format. It's +identical to the normal Forwarded Log except the name of the virtual host +is prepended to each line. -### EXIT STATUS +Use the appropriate script for the statistic and for the log format: -The **flv** utility exits 0 on success, and >0 if an error occurs. +### Pages + $ flp /var/www/access.log + /docs.html 641 + / 3227 -### EXAMPLES +Use `sort -nrk 2` to sort on second column: -Print all -*views* -from -*/var/www/logs/access.log:* + $ clp /var/www/access.log | sort -nrk 2 + / 3227 + /docs.html 641 - $ flv /var/www/logs/access.log +### Uniques and Visits -### SEE ALSO +Uniques and visits scripts output the actual unique and visit requests +by default. Use `wc -l` to count: -awk(1) -httpd(8) -httpd.conf(5) -clv(1) -cblv(1) + $ flu /var/www/access.log | wc -l + 2000 -## flu(1) +### Referrers -### NAME + $ flr /var/www/access.log + none 865 + https://example.com 1222 -**flu** - forwarded log uniques +Use `sort -nrk 2` to sort on second column: -### SYNOPSIS + $ flr /var/www/access.log | sort -nrk 2 + https://example.com 1222 + none 865 -**flu** -\[*file ...*] +We recommend using [cblfb](http://adi.onl/cblfb.html) for +filtering bots. -### DESCRIPTION + $ flr /var/www/access.log | cblfb + none 821 + https://example.com 1212 -The -**flu** -utility reads -each -\[*file ...*] -in -*Forwarded Log Format* -and prints only the -*uniques* -to standard output. +Also, it's a good idea to enable log rotation. -*uniques* -are defined as unique remote hosts -that made at least one -*GET* -request to paths ending in -*/* -or -*.html* -that returned a -*200* -HTTP response status code. +The scripts operate on `GET` requests to paths with no extension or +ending in `.html` that returned a 200 HTTP response status code. -If -\[*file ...*] -is omitted, -**flu** -reads from standard input. - -### EXIT STATUS - -The **flu** utility exits 0 on success, and >0 if an error occurs. - -### EXAMPLES - -Print all -*uniques* -from -*/var/www/logs/access.log:* - - $ flu /var/www/logs/access.log - -### SEE ALSO - -awk(1) -httpd(8) -httpd.conf(5) -clu(1) -cblu(1) - -## flp(1) - -### NAME - -**flp** - forwarded log pages - -### SYNOPSIS - -**flp** -\[*file ...*] - -### DESCRIPTION - -The -**flp** -utility reads -each -\[*file ...*] -in -*Forwarded Log Format* -and prints a list of -*pages* -to standard output including the request count. - -*pages* -are defined as -*paths* -ending in -*/* -or -*.html* -to which -*GET* -requests have been made that returned a -*200* -HTTP response status code. - -If -\[*file ...*] -is omitted, -**flp** -reads from standard input. - -### EXIT STATUS - -The **flp** utility exits 0 on success, and >0 if an error occurs. - -### EXAMPLES - -Print all -*pages* -and their respective request count -from -*/var/www/logs/access.log*: - - $ flp /var/www/logs/access.log - -### SEE ALSO - -awk(1) -httpd(8) -httpd.conf(5) -clp(1) -cblp(1) - -## flr(1) - -### NAME - -**flr** - forwarded log referrer - -### SYNOPSIS - -**flr** -\[*file ...*] - -### DESCRIPTION - -The -**flr** -utility reads -each -\[*file ...*] -in -*Forwarded Log Format* -and prints a list of -*referrers* -to standard output including the request count. - -*referrers* -are defined as the address of the webpage which is linked to a -*path* -ending in -*/* -or -*.html* -to which -*GET* -requests have been made that returned a -*200* -HTTP response status code. - -If -\[*file ...*] -is omitted, -**flr** -reads from standard input. - -### EXIT STATUS - -The **flr** utility exits 0 on success, and >0 if an error occurs. - -### EXAMPLES - -Print all -*referrers* -and their respective request count -from -*/var/www/logs/access.log*: - - $ flr /var/www/logs/access.log - -### SEE ALSO - -awk(1) -httpd(8) -httpd.conf(5) -cblr(1) - -## flvv(1) - -### NAME - -**flvv** - forwarded log (virtual host) views - -### SYNOPSIS - -**flvv** -\[*file ...*] - -### DESCRIPTION - -The -**flvv** -utility reads -each -\[*file ...*] -in -*Forwarded Log Format with Virtual Host* -and prints only the -*views* -to standard output. - -*views* -are defined as -*GET* -requests to paths ending in -*/* -or -*.html* -that returned a -*200* -HTTP response status code. - -If -\[*file ...*] -is omitted, -**flvv** -reads from standard input. - -### EXIT STATUS - -The **flvv** utility exits 0 on success, and >0 if an error occurs. - -### EXAMPLES - -Print all -*views* -from -*/var/www/logs/access.log:* - - $ flvv /var/www/logs/access.log - -### SEE ALSO - -awk(1) -httpd(8) -httpd.conf(5) -clvv(1) -cblvv(1) - -## flvu(1) - -### NAME - -**flvu** - forwarded log (virtual host) uniques - -### SYNOPSIS - -**flvu** -\[*file ...*] - -### DESCRIPTION - -The -**flvu** -utility reads -each -\[*file ...*] -in -*Forwarded Log Format with Virtual Hosts* -and prints only the -*uniques* -to standard output. - -*uniques* -to standard output. - -*uniques* -are defined as unique remote hosts -that made at least one -*GET* -request -for files with an -*.html* -extension, including the -*/* -path that returned a -*200* -HTTP response status code. - -If -\[*file ...*] -is omitted, -**flvu** -reads from standard input. - -### EXIT STATUS - -The **flvu** utility exits 0 on success, and >0 if an error occurs. - -### EXAMPLES - -Print all -*uniques* -from -*/var/www/logs/access.log:* - - $ flvu /var/www/logs/access.log - -### SEE ALSO - -awk(1) -httpd(8) -httpd.conf(5) -clvu(1) -cblvu(1) - -## flvp(1) - -### NAME - -**flvp** - forwarded log (virtual host) pages - -### SYNOPSIS - -**flvp** -\[*file ...*] - -### DESCRIPTION - -The -**flvp** -utility reads -each -\[*file ...*] -in -*Forwarded Log Format with Virtual Host* -and prints a list of -*pages* -to standard output including the request count. - -*pages* -are defined as -*paths* -ending in -*/* -or -*.html* -to which -*GET* -requests have been made that returned a -*200* -HTTP response status code. - -If -\[*file ...*] -is omitted, -**flvp** -reads from standard input. - -### EXIT STATUS - -The **flvp** utility exits 0 on success, and >0 if an error occurs. - -### EXAMPLES - -Print all -*pages* -and their respective request count -from -*/var/www/logs/access.log*: - - $ flvp /var/www/logs/access.log - -### SEE ALSO - -awk(1) -httpd(8) -httpd.conf(5) -clvp(1) -cblvp(1) - -## flvr(1) - -### NAME - -**flvr** - forwarded log (virtual host) referrer - -### SYNOPSIS - -**flvr** -\[*file ...*] - -### DESCRIPTION - -The -**flvr** -utility reads -each -\[*file ...*] -in -*Forwarded Log Format Virtual Host* -and prints a list of -*referrers* -to standard output including the request count. - -*referrers* -are defined as the address of the webpage which is linked to a -*path* -ending in -*/* -or -*.html* -to which -*GET* -requests have been made that returned a -*200* -HTTP response status code. - -If -\[*file ...*] -is omitted, -**flvr** -reads from standard input. - -### EXIT STATUS - -The **flvr** utility exits 0 on success, and >0 if an error occurs. - -### EXAMPLES - -Print all -*referrers* -and their respective request count -from -*/var/www/logs/access.log*: - - $ flvr /var/www/logs/access.log - -### SEE ALSO - -awk(1) -httpd(8) -httpd.conf(5) -cblvr(1) + https://example.com/docs + https://example.com/docs/ + https://examples.com/docs.html diff -urPN 0.1.0/flp 1.0.0/flp --- 0.1.0/flp Mon Aug 10 21:18:03 2020 +++ 1.0.0/flp Fri Mar 18 15:52:41 2022 @@ -1,7 +1,13 @@ -#!/bin/sh - -awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/c.awk \ --f "${CBLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cbl}"/cb.awk \ --f "${FLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/fl}"/f.awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/p.awk "$@" +#!/usr/bin/awk -f +$6 !~ /GET/ || +$9 != 200 { +next +} +$7 !~ /\/*\.[^.]*$/ || +$7 ~ /\.html$/ { +ps[$7]++ +} +END { +for (p in ps) +print p" "ps[p] +} diff -urPN 0.1.0/flp.1 1.0.0/flp.1 --- 0.1.0/flp.1 Mon Aug 10 21:18:03 2020 +++ 1.0.0/flp.1 Fri Mar 18 15:52:41 2022 @@ -22,9 +22,7 @@ .Em pages are defined as .Em paths -ending in -.Pa / -or +with no extension or ending in .Pa .html to which .Em GET diff -urPN 0.1.0/flr 1.0.0/flr --- 0.1.0/flr Mon Aug 10 21:18:03 2020 +++ 1.0.0/flr Fri Mar 18 15:52:41 2022 @@ -1,7 +1,16 @@ -#!/bin/sh - -awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/c.awk \ --f "${CBLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cbl}"/cb.awk \ --f "${FLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/fl}"/f.awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/r.awk "$@" +#!/usr/bin/awk -f +$6 !~ /GET/ || +$9 != 200 { +next +} +{ +gsub("\"", "", $11) +if ($11 == "") $11="none" +} +$7 !~ /\/*\.[^.]*$/ || +$7 ~ /\.html$/ { +rs[$11]++ +} +END { +for (r in rs) print r" "rs[r] +} diff -urPN 0.1.0/flr.1 1.0.0/flr.1 --- 0.1.0/flr.1 Mon Aug 10 21:18:03 2020 +++ 1.0.0/flr.1 Fri Mar 18 15:52:41 2022 @@ -22,9 +22,7 @@ .Em referrers are defined as the address of the webpage which is linked to a .Em path -ending in -.Pa / -or +with no extension or ending in .Pa .html to which .Em GET diff -urPN 0.1.0/flu 1.0.0/flu --- 0.1.0/flu Mon Aug 10 21:18:03 2020 +++ 1.0.0/flu Fri Mar 18 15:52:41 2022 @@ -1,7 +1,8 @@ -#!/bin/sh - -awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/c.awk \ --f "${CBLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cbl}"/cb.awk \ --f "${FLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/fl}"/f.awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/u.awk "$@" +#!/usr/bin/awk -f +$6 !~ /GET/ || +$9 != 200 { +next +} +($7 !~ /\/*\.[^.]*$/ || +$7 ~ /\.html$/) && +!us[$(NF-1)]++ diff -urPN 0.1.0/flu.1 1.0.0/flu.1 --- 0.1.0/flu.1 Mon Aug 10 21:18:03 2020 +++ 1.0.0/flu.1 Fri Mar 18 15:52:41 2022 @@ -23,9 +23,7 @@ are defined as unique remote hosts that made at least one .Em GET -request to paths ending in -.Pa / -or +request to paths with no extension or ending in .Pa .html that returned a .Em 200 diff -urPN 0.1.0/flv 1.0.0/flv --- 0.1.0/flv Mon Aug 10 21:18:03 2020 +++ 1.0.0/flv Fri Mar 18 15:52:41 2022 @@ -1,7 +1,7 @@ -#!/bin/sh - -awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/c.awk \ --f "${CBLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cbl}"/cb.awk \ --f "${FLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/fl}"/f.awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/v.awk "$@" +#!/usr/bin/awk -f +$6 !~ /GET/ || +$9 != 200 { +next +} +$7 !~ /\/*\.[^.]*$/ || +$7 ~ /\.html$/ diff -urPN 0.1.0/flv.1 1.0.0/flv.1 --- 0.1.0/flv.1 Mon Aug 10 21:18:03 2020 +++ 1.0.0/flv.1 Fri Mar 18 15:52:41 2022 @@ -23,9 +23,7 @@ are defined as .Em GET requests -to paths ending in -.Pa / -or +to paths with no extension ending in .Pa .html that returned a .Em 200 diff -urPN 0.1.0/flvp 1.0.0/flvp --- 0.1.0/flvp Mon Aug 10 21:18:03 2020 +++ 1.0.0/flvp Fri Mar 18 15:52:41 2022 @@ -1,7 +1,13 @@ -#!/bin/sh - -awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/cv.awk \ --f "${CBLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cbl}"/cbv.awk \ --f "${FLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/fl}"/fv.awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/p.awk "$@" +#!/usr/bin/awk -f +$7 !~ /GET/ || +$10 != 200 { +next +} +$8 !~ /\/*\.[^.]*$/ || +$8 ~ /\.html$/ { +ps[$8]++ +} +END { +for (p in ps) +print p" "ps[p] +} diff -urPN 0.1.0/flvp.1 1.0.0/flvp.1 --- 0.1.0/flvp.1 Mon Aug 10 21:18:03 2020 +++ 1.0.0/flvp.1 Fri Mar 18 15:52:41 2022 @@ -22,9 +22,7 @@ .Em pages are defined as .Em paths -ending in -.Pa / -or +with no extension or ending in .Pa .html to which .Em GET diff -urPN 0.1.0/flvr 1.0.0/flvr --- 0.1.0/flvr Mon Aug 10 21:18:03 2020 +++ 1.0.0/flvr Fri Mar 18 15:52:41 2022 @@ -1,7 +1,16 @@ -#!/bin/sh - -awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/cv.awk \ --f "${CBLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cbl}"/cbv.awk \ --f "${FLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/fl}"/fv.awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cbl}"/r.awk "$@" +#!/usr/bin/awk -f +$7 !~ /GET/ || +$10 != 200 { +next +} +{ +gsub("\"", "", $12) +if ($12 == "") $12="none" +} +$8 !~ /\/*\.[^.]*$/ || +$8 ~ /\.html$/ { +rs[$12]++ +} +END { +for (r in rs) print r" "rs[r] +} diff -urPN 0.1.0/flvr.1 1.0.0/flvr.1 --- 0.1.0/flvr.1 Mon Aug 10 21:18:03 2020 +++ 1.0.0/flvr.1 Fri Mar 18 15:52:41 2022 @@ -22,9 +22,7 @@ .Em referrers are defined as the address of the webpage which is linked to a .Em path -ending in -.Pa / -or +with no extension or ending in .Pa .html to which .Em GET diff -urPN 0.1.0/flvu 1.0.0/flvu --- 0.1.0/flvu Mon Aug 10 21:18:03 2020 +++ 1.0.0/flvu Fri Mar 18 15:52:41 2022 @@ -1,7 +1,8 @@ -#!/bin/sh - -awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/cv.awk \ --f "${CBLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cbl}"/cbv.awk \ --f "${FLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/fl}"/fv.awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/u.awk "$@" +#!/usr/bin/awk -f +$7 !~ /GET/ || +$10 != 200 { +next +} +($8 !~ /\/*\.[^.]*$/ || +$8 ~ /\.html$/) && +!us[$(NF-1)]++ diff -urPN 0.1.0/flvv 1.0.0/flvv --- 0.1.0/flvv Mon Aug 10 21:18:03 2020 +++ 1.0.0/flvv Fri Mar 18 15:52:41 2022 @@ -1,7 +1,7 @@ -#!/bin/sh - -awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/cv.awk \ --f "${CBLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cbl}"/cbv.awk \ --f "${FLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/fl}"/fv.awk \ --f "${CLSHAREPATH:-$(dirname "$(realpath "$0")")/../share/cl}"/v.awk "$@" +#!/usr/bin/awk -f +$7 !~ /GET/ || +$10 != 200 { +next +} +$8 !~ /\/*\.[^.]*$/ || +$8 ~ /\.html$/ diff -urPN 0.1.0/flvv.1 1.0.0/flvv.1 --- 0.1.0/flvv.1 Mon Aug 10 21:18:03 2020 +++ 1.0.0/flvv.1 Fri Mar 18 15:52:41 2022 @@ -22,9 +22,7 @@ .Em views are defined as .Em GET -requests to paths ending in -.Pa / -or +requests to paths with no extension or ending in .Pa .html that returned a .Em 200 diff -urPN 0.1.0/share/f.awk 1.0.0/share/f.awk --- 0.1.0/share/f.awk Mon Aug 10 21:18:03 2020 +++ 1.0.0/share/f.awk Thu Jan 1 01:00:00 1970 @@ -1,4 +0,0 @@ -{ -h=$(NF-1) -fp=$NF -} diff -urPN 0.1.0/share/fv.awk 1.0.0/share/fv.awk --- 0.1.0/share/fv.awk Mon Aug 10 21:18:03 2020 +++ 1.0.0/share/fv.awk Thu Jan 1 01:00:00 1970 @@ -1,4 +0,0 @@ -{ -h=$(NF-1) -fp=$NF -}