ISTI Open Access

#!/usr/bin/python

# vim: et sw=4 ts=4:

# -*- coding: utf-8 -*-

#

# Matomo - free/libre analytics platform

#

# @link https://matomo.org

# @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later

# @version $Id$

#

# For more info see: https://matomo.org/log-analytics/ and https://matomo.org/docs/log-analytics-tool-how-to/

#

# Requires Python 3.5, 3.6, 3.7 or higher

#

from __future__ import print_function  # this is needed that python2 can run the script until the warning below

import sys

if sys.version_info[0] != 3:

    print('OpenAIRE Generic tracker does not support Python 2 any more.')

    print('Please use Python 3.5, 3.6, 3.7 or 3.8')

    sys.exit(1)

import bz2

import datetime

import gzip

import http.client as httplib

import inspect

import itertools

import logging

import os

import os.path

import queue as queue

import re

import sys

import threading

import time

import urllib.request as urllib2

import urllib.parse as urlparse

import traceback

import socket

import textwrap

import yaml

import getopt

try:

    import json

except ImportError:

    try:

        import simplejson as json

    except ImportError:

        if sys.version_info < (2, 6):

            print >> sys.stderr, 'simplejson (http://pypi.python.org/pypi/simplejson/) is required.'

            sys.exit(1)

##

## Constants.

##

MATOMO_DEFAULT_MAX_ATTEMPTS = 3

MATOMO_DEFAULT_DELAY_AFTER_FAILURE = 10

DEFAULT_SOCKET_TIMEOUT = 300

##

## Formats.

##

class BaseFormatException(Exception): pass

class BaseFormat(object):

    def __init__(self, name):

        self.name = name

        self.regex = None

        self.date_format = '%d/%b/%Y:%H:%M:%S'

    def check_format(self, file):

        line = file.readline()

        try:

            file.seek(0)

        except IOError:

            pass

        return self.check_format_line(line)

    def check_format_line(self, line):

        return False

class JsonFormat(BaseFormat):

    def __init__(self, name):

        super(JsonFormat, self).__init__(name)

        self.json = None

        self.date_format = '%Y-%m-%dT%H:%M:%S'

    def check_format_line(self, line):

        try:

            self.json = json.loads(line)

            return True

        except:

            return False

    def match(self, line):

        try:

            # nginx outputs malformed JSON w/ hex escapes when confronted w/ non-UTF input. we have to

            # workaround this by converting hex escapes in strings to unicode escapes. the conversion is naive,

            # so it does not take into account the string's actual encoding (which we don't have access to).

            line = line.replace('\\x', '\\u00')

            self.json = json.loads(line)

            return self

        except:

            self.json = None

            return None

    def get(self, key):

        # Some ugly patchs ...

        if key == 'generation_time_milli':

            self.json[key] =  int(float(self.json[key]) * 1000)

        # Patch date format ISO 8601

        elif key == 'date':

            tz = self.json[key][19:]

            self.json['timezone'] = tz.replace(':', '')

            self.json[key] = self.json[key][:19]

        try:

            return self.json[key]

        except KeyError:

            raise BaseFormatException()

    def get_all(self,):

        return self.json

    def remove_ignored_groups(self, groups):

        for group in groups:

            del self.json[group]

class RegexFormat(BaseFormat):

    def __init__(self, name, regex, date_format=None):

        super(RegexFormat, self).__init__(name)

        if regex is not None:

            self.regex = re.compile(regex)

        if date_format is not None:

            self.date_format = date_format

        self.matched = None

    def check_format_line(self, line):

        return self.match(line)

    def match(self,line):

        if not self.regex:

            return None

        match_result = self.regex.match(line)

        if match_result:

            self.matched = match_result.groupdict()

        else:

            self.matched = None

        return match_result

    def get(self, key):

        try:

            return self.matched[key]

        except KeyError:

            raise BaseFormatException("Cannot find group '%s'." % key)

    def get_all(self,):

        return self.matched

    def remove_ignored_groups(self, groups):

        for group in groups:

            del self.matched[group]

class W3cExtendedFormat(RegexFormat):

    FIELDS_LINE_PREFIX = '#Fields: '

    fields = {

        'date': '(?P<date>\d+[-\d+]+',

        'time': '[\d+:]+)[.\d]*?', # TODO should not assume date & time will be together not sure how to fix ATM.

        'cs-uri-stem': '(?P<path>/\S*)',

        'cs-uri-query': '(?P<query_string>\S*)',

        'c-ip': '"?(?P<ip>[\w*.:-]*)"?',

        'cs(User-Agent)': '(?P<user_agent>".*?"|\S*)',

        'cs(Referer)': '(?P<referrer>\S+)',

        'sc-status': '(?P<status>\d+)',

        'sc-bytes': '(?P<length>\S+)',

        'cs-host': '(?P<host>\S+)',

        'cs-method': '(?P<method>\S+)',

        'cs-username': '(?P<userid>\S+)',

        'time-taken': '(?P<generation_time_secs>[.\d]+)'

    }

    def __init__(self):

        super(W3cExtendedFormat, self).__init__('w3c_extended', None, '%Y-%m-%d %H:%M:%S')

    def check_format(self, file):

        self.create_regex(file)

        # if we couldn't create a regex, this file does not follow the W3C extended log file format

        if not self.regex:

            try:

                file.seek(0)

            except IOError:

                pass

            return

        first_line = file.readline()

        try:

            file.seek(0)

        except IOError:

            pass

        return self.check_format_line(first_line)

    def create_regex(self, file):

        fields_line = None

        #if config.options.w3c_fields:

        #    fields_line = config.options.w3c_fields

        # collect all header lines up until the Fields: line

        # if we're reading from stdin, we can't seek, so don't read any more than the Fields line

        header_lines = []

        while fields_line is None:

            line = file.readline().strip()

            if not line:

                continue

            if not line.startswith('#'):

                break

            if line.startswith(W3cExtendedFormat.FIELDS_LINE_PREFIX):

                fields_line = line

            else:

                header_lines.append(line)

        if not fields_line:

            return

        # store the header lines for a later check for IIS

        self.header_lines = header_lines

        # Parse the 'Fields: ' line to create the regex to use

        full_regex = []

        expected_fields = type(self).fields.copy() # turn custom field mapping into field => regex mapping

        # if the --w3c-time-taken-millisecs option is used, make sure the time-taken field is interpreted as milliseconds

        #if config.options.w3c_time_taken_in_millisecs:

        #    expected_fields['time-taken'] = '(?P<generation_time_milli>[\d.]+)'

        for mapped_field_name, field_name in config.options.custom_w3c_fields.iteritems():

            expected_fields[mapped_field_name] = expected_fields[field_name]

            del expected_fields[field_name]

        # add custom field regexes supplied through --w3c-field-regex option

        #for field_name, field_regex in config.options.w3c_field_regexes.iteritems():

        #    expected_fields[field_name] = field_regex

        # Skip the 'Fields: ' prefix.

        fields_line = fields_line[9:].strip()

        for field in re.split('\s+', fields_line):

            try:

                regex = expected_fields[field]

            except KeyError:

                regex = '(?:".*?"|\S+)'

            full_regex.append(regex)

        full_regex = '\s+'.join(full_regex)

        logging.debug("Based on 'Fields:' line, computed regex to be %s", full_regex)

        self.regex = re.compile(full_regex)

    def check_for_iis_option(self):

       logging.info("WARNING: IIS log file being parsed without --w3c-time-taken-milli option. IIS"

                         " stores millisecond values in the time-taken field. If your logfile does this, the aforementioned"

                         " option must be used in order to get accurate generation times.")

    def _is_iis(self):

        return len([line for line in self.header_lines if 'internet information services' in line.lower() or 'iis' in line.lower()]) > 0

    def _is_time_taken_milli(self):

        return 'generation_time_milli' not in self.regex.pattern

class IisFormat(W3cExtendedFormat):

    fields = W3cExtendedFormat.fields.copy()

    fields.update({

        'time-taken': '(?P<generation_time_milli>[.\d]+)',

        'sc-win32-status': '(?P<__win32_status>\S+)' # this group is useless for log importing, but capturing it

                                                     # will ensure we always select IIS for the format instead of

                                                     # W3C logs when detecting the format. This way there will be

                                                     # less accidental importing of IIS logs w/o --w3c-time-taken-milli.

    })

    def __init__(self):

        super(IisFormat, self).__init__()

        self.name = 'iis'

class ShoutcastFormat(W3cExtendedFormat):

    fields = W3cExtendedFormat.fields.copy()

    fields.update({

        'c-status': '(?P<status>\d+)',

        'x-duration': '(?P<generation_time_secs>[.\d]+)'

    })

    def __init__(self):

        super(ShoutcastFormat, self).__init__()

        self.name = 'shoutcast'

    def get(self, key):

        if key == 'user_agent':

            user_agent = super(ShoutcastFormat, self).get(key)

            return urllib2.unquote(user_agent)

        else:

            return super(ShoutcastFormat, self).get(key)

class AmazonCloudFrontFormat(W3cExtendedFormat):

    fields = W3cExtendedFormat.fields.copy()

    fields.update({

        'x-event': '(?P<event_action>\S+)',

        'x-sname': '(?P<event_name>\S+)',

        'cs-uri-stem': '(?:rtmp:/)?(?P<path>/\S*)',

        'c-user-agent': '(?P<user_agent>".*?"|\S+)',

        # following are present to match cloudfront instead of W3C when we know it's cloudfront

        'x-edge-location': '(?P<x_edge_location>".*?"|\S+)',

        'x-edge-result-type': '(?P<x_edge_result_type>".*?"|\S+)',

        'x-edge-request-id': '(?P<x_edge_request_id>".*?"|\S+)',

        'x-host-header': '(?P<x_host_header>".*?"|\S+)'

    })

    def __init__(self):

        super(AmazonCloudFrontFormat, self).__init__()

        self.name = 'amazon_cloudfront'

    def get(self, key):

        if key == 'event_category' and 'event_category' not in self.matched:

            return 'cloudfront_rtmp'

        elif key == 'status' and 'status' not in self.matched:

            return '200'

        elif key == 'user_agent':

            user_agent = super(AmazonCloudFrontFormat, self).get(key)

            return urllib2.unquote(user_agent)

        else:

            return super(AmazonCloudFrontFormat, self).get(key)

_HOST_PREFIX = '(?P<host>[\w\-\.]*)(?::\d+)?\s+'

_COMMON_LOG_FORMAT = (

    '(?P<ip>[\w*.:-]+)\s+\S+\s+(?P<userid>\S+)\s+\[(?P<date>.*?)\s+(?P<timezone>.*?)\]\s+'

    '"(?P<method>\S+)\s+(?P<path>.*?)\s+\S+"\s+(?P<status>\d+)\s+(?P<length>\S+)'

)

_NCSA_EXTENDED_LOG_FORMAT = (_COMMON_LOG_FORMAT +

    '\s+"(?P<referrer>.*?)"\s+"(?P<user_agent>.*?)"'

)

_S3_LOG_FORMAT = (

    '\S+\s+(?P<host>\S+)\s+\[(?P<date>.*?)\s+(?P<timezone>.*?)\]\s+(?P<ip>[\w*.:-]+)\s+'

    '(?P<userid>\S+)\s+\S+\s+\S+\s+\S+\s+"(?P<method>\S+)\s+(?P<path>.*?)\s+\S+"\s+(?P<status>\d+)\s+\S+\s+(?P<length>\S+)\s+'

    '\S+\s+\S+\s+\S+\s+"(?P<referrer>.*?)"\s+"(?P<user_agent>.*?)"'

)

_ICECAST2_LOG_FORMAT = ( _NCSA_EXTENDED_LOG_FORMAT +

    '\s+(?P<session_time>[0-9-]+)'

)

_ELB_LOG_FORMAT = (

    '(?P<date>[0-9-]+T[0-9:]+)\.\S+\s+\S+\s+(?P<ip>[\w*.:-]+):\d+\s+\S+:\d+\s+\S+\s+(?P<generation_time_secs>\S+)\s+\S+\s+'

    '(?P<status>\d+)\s+\S+\s+\S+\s+(?P<length>\S+)\s+'

    '"\S+\s+\w+:\/\/(?P<host>[\w\-\.]*):\d+(?P<path>\/\S*)\s+[^"]+"\s+"(?P<user_agent>[^"]+)"\s+\S+\s+\S+'

)

_OVH_FORMAT = (

    '(?P<ip>\S+)\s+' + _HOST_PREFIX + '(?P<userid>\S+)\s+\[(?P<date>.*?)\s+(?P<timezone>.*?)\]\s+'

    '"\S+\s+(?P<path>.*?)\s+\S+"\s+(?P<status>\S+)\s+(?P<length>\S+)'

    '\s+"(?P<referrer>.*?)"\s+"(?P<user_agent>.*?)"'

)

FORMATS = {

    'common': RegexFormat('common', _COMMON_LOG_FORMAT),

    'common_vhost': RegexFormat('common_vhost', _HOST_PREFIX + _COMMON_LOG_FORMAT),

    'ncsa_extended': RegexFormat('ncsa_extended', _NCSA_EXTENDED_LOG_FORMAT),

    'common_complete': RegexFormat('common_complete', _HOST_PREFIX + _NCSA_EXTENDED_LOG_FORMAT),

    'w3c_extended': W3cExtendedFormat(),

    'amazon_cloudfront': AmazonCloudFrontFormat(),

    'iis': IisFormat(),

    'shoutcast': ShoutcastFormat(),

    's3': RegexFormat('s3', _S3_LOG_FORMAT),

    'icecast2': RegexFormat('icecast2', _ICECAST2_LOG_FORMAT),

    'elb': RegexFormat('elb', _ELB_LOG_FORMAT, '%Y-%m-%dT%H:%M:%S'),

    'nginx_json': JsonFormat('nginx_json'),

    'ovh': RegexFormat('ovh', _OVH_FORMAT)

}

##

## Code.

##

class Configuration(object):

    """

    Stores all the configuration options by reading sys.argv and parsing,

    if needed, the config.inc.php.

    It has 2 attributes: options and filenames.

    """

    class Error(Exception):

        pass

    def _create_parser(self):

        matomoConfig = None

        with open("matomo_config.yaml", 'r') as stream:

            try:

                matomoConfig=yaml.load(stream, Loader=yaml.FullLoader)

            except yaml.YAMLError as exc:

                print(exc)

        """

        Initialize and return the OptionParser instance.

        """

        self.options = matomoConfig

        return self.options

    def _parse_args(self, options):

        """

        Parse the command line args and create self.options and self.filenames.

        """

        filePath = os.path.abspath(os.path.abspath(sys.argv[-1]))

        if os.path.isdir(filePath):

            self.filenames  = [(filePath+"/"+x) for x in os.listdir(filePath)]

        elif os.path.isfile(filePath):

            self.filenames = [filePath]

        # Configure logging before calling logging.{debug,info}.

        logging.basicConfig(

            format='%(asctime)s: [%(levelname)s] %(message)s',

            filename='Matomo_import.log',

            level=logging.INFO,

        )

    def __init__(self):

        self._parse_args(self._create_parser())

    def get_resolver(self):

        if self.options.site_id:

            logging.debug('Resolver: static')

            return StaticResolver(self.options.site_id)

        else:

            logging.debug('Resolver: dynamic')

            return DynamicResolver()

class UrlHelper(object):

    @staticmethod

    def convert_array_args(args):

        """

        Converts PHP deep query param arrays (eg, w/ names like hsr_ev[abc][0][]=value) into a nested list/dict

        structure that will convert correctly to JSON.

        """

        final_args = {}

        for key, value in args.items():

            indices = key.split('[')

            if '[' in key:

                # contains list of all indices, eg for abc[def][ghi][] = 123, indices would be ['abc', 'def', 'ghi', '']

                indices = [i.rstrip(']') for i in indices]

                # navigate the multidimensional array final_args, creating lists/dicts when needed, using indices

                element = final_args

                for i in range(0, len(indices) - 1):

                    idx = indices[i]

                    # if there's no next key, then this element is a list, otherwise a dict

                    element_type = list if not indices[i + 1] else dict

                    if idx not in element or not isinstance(element[idx], element_type):

                        element[idx] = element_type()

                    element = element[idx]

                # set the value in the final container we navigated to

                if not indices[-1]: # last indice is '[]'

                    element.append(value)

                else: # last indice has a key, eg, '[abc]'

                    element[indices[-1]] = value

            else:

                final_args[key] = value

        return UrlHelper._convert_dicts_to_arrays(final_args)

    @staticmethod

    def _convert_dicts_to_arrays(d):

        # convert dicts that have contiguous integer keys to arrays

        for key, value in d.items():

            if not isinstance(value, dict):

                continue

            if UrlHelper._has_contiguous_int_keys(value):

                d[key] = UrlHelper._convert_dict_to_array(value)

            else:

                d[key] = UrlHelper._convert_dicts_to_arrays(value)

        return d

    @staticmethod

    def _has_contiguous_int_keys(d):

        for i in range(0, len(d)):

            if str(i) not in d:

                return False

        return True

    @staticmethod

    def _convert_dict_to_array(d):

        result = []

        for i in range(0, len(d)):

            result.append(d[str(i)])

        return result

class Matomo(object):

    """

    Make requests to Matomo.

    """

    class Error(Exception):

        def __init__(self, message, code = None):

            super(Exception, self).__init__(message)

            self.code = code

    class RedirectHandlerWithLogging(urllib2.HTTPRedirectHandler):

        """

        Special implementation of HTTPRedirectHandler that logs redirects in debug mode

        to help users debug system issues.

        """

        def redirect_request(self, req, fp, code, msg, hdrs, newurl):

            logging.debug("Request redirected (code: %s) to '%s'" % (code, newurl))

            return urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, hdrs, newurl)

    @staticmethod

    def _call(path, args, headers=None, url=None, data=None):

        """

        Make a request to the Matomo site. It is up to the caller to format

        arguments, to embed authentication, etc.

        """

        if url is None:

            url = config.options['Matomo_Parameters']['matomo_url']

        headers = headers or {}

        if data is None:

            # If Content-Type isn't defined, PHP do not parse the request's body.

            headers['Content-type'] = 'application/x-www-form-urlencoded'

            data = urllib.urlencode(args)

        elif not isinstance(data, str) and headers['Content-type'] == 'application/json':

            data = json.dumps(data).encode("utf-8")

            if args:

                path = path + '?' + urllib.urlencode(args)

        headers['User-Agent'] = 'Matomo/LogImport'

        try:

            timeout = config.options['Matomo_Parameters']['default_socket_timeout']

        except:

            timeout = None # the config global object may not be created at this point

        request = urllib2.Request(url + path, data, headers)

        # Use non-default SSL context if invalid certificates shall be

        # accepted.

        '''

        if config.options.accept_invalid_ssl_certificate and \

                sys.version_info >= (2, 7, 9):

            ssl_context = ssl.create_default_context()

            ssl_context.check_hostname = False

            ssl_context.verify_mode = ssl.CERT_NONE

            https_handler_args = {'context': ssl_context}

        else:

            https_handler_args = {}

        opener = urllib2.build_opener(

            Matomo.RedirectHandlerWithLogging(),

            urllib2.HTTPSHandler(**https_handler_args))

        response = opener.open(request, timeout = timeout)

        result = response.read()

        response.close()

        return result

        '''

        https_handler_args = {}

        opener = urllib2.build_opener(

            Matomo.RedirectHandlerWithLogging(),

            urllib2.HTTPSHandler(**https_handler_args))

        response = opener.open(request, timeout = timeout)

        result = response.read()

        response.close()

        return result

    @staticmethod

    def _call_api(method, **kwargs):

        """

        Make a request to the Matomo API taking care of authentication, body

        formatting, etc.

        """

        args = {

            'module' : 'API',

            'format' : 'json2',

            'method' : method,

            'filter_limit' : '-1',

        }

        if kwargs:

            args.update(kwargs)

        # Convert lists into appropriate format.

        # See: http://developer.matomo.org/api-reference/reporting-api#passing-an-array-of-data-as-a-parameter

        # Warning: we have to pass the parameters in order: foo[0], foo[1], foo[2]

        # and not foo[1], foo[0], foo[2] (it will break Matomo otherwise.)

        final_args = []

        for key, value in args.items():

            if isinstance(value, (list, tuple)):

                for index, obj in enumerate(value):

                    final_args.append(('%s[%d]' % (key, index), obj))

            else:

                final_args.append((key, value))

#        logging.debug('%s' % final_args)

#        logging.debug('%s' % url)

        res = Matomo._call('/', final_args, url=url)

        try:

            return json.loads(res)

        except ValueError:

            raise urllib2.URLError('Matomo returned an invalid response: ' + res)

    @staticmethod

    def _call_wrapper(func, expected_response, on_failure, *args, **kwargs):

        """

        Try to make requests to Matomo at most MATOMO_FAILURE_MAX_RETRY times.

        """

        errors = 0

        while True:

            try:

                response = func(*args, **kwargs)

                if expected_response is not None and response != expected_response:

                    if on_failure is not None:

                        error_message = on_failure(response, kwargs.get('data'))

                    else:

                        error_message = "didn't receive the expected response. Response was %s " % response

                    raise urllib2.URLError(error_message)

                return response

            except (urllib2.URLError, httplib.HTTPException, ValueError, socket.timeout) as e:

                logging.info('Error when connecting to Matomo: %s', e)

                code = None

                if isinstance(e, urllib2.HTTPError):

                    # See Python issue 13211.

                    message = 'HTTP Error %s %s' % (e.code, e.msg)

                    code = e.code

                elif isinstance(e, urllib2.URLError):

                    message = e.reason

                else:

                    message = str(e)

                # decorate message w/ HTTP response, if it can be retrieved

                if hasattr(e, 'read'):

                    message = message + ", response: " + e.read()

                try:

                    delay_after_failure = config.options["Matomo_Parameters"]["delay_after_failure"]

                    max_attempts = config.options["Matomo_Parameters"]["default_max_attempts"]

                except NameError:

                    delay_after_failure = MATOMO_DEFAULT_DELAY_AFTER_FAILURE

                    max_attempts = MATOMO_DEFAULT_MAX_ATTEMPTS

                errors += 1

                if errors == max_attempts:

                    logging.info("Max number of attempts reached, server is unreachable!")

                    raise Matomo.Error(message, code)

                else:

                    logging.info("Retrying request, attempt number %d" % (errors + 1))

                    time.sleep(delay_after_failure)

    @classmethod

    def call(cls, path, args, expected_content=None, headers=None, data=None, on_failure=None):

        return cls._call_wrapper(cls._call, expected_content, on_failure, path, args, headers,

                                    data=data)

    @classmethod

    def call_api(cls, method, **kwargs):

        return cls._call_wrapper(cls._call_api, None, None, method, **kwargs)

class Recorder(object):

    """

    A Recorder fetches hits from the Queue and inserts them into Matomo using

    the API.

    """

    recorders = []

    def __init__(self):

        self.queue = queue.Queue(maxsize=2)

        # if bulk tracking disabled, make sure we can store hits outside of the Queue

        #if not config.options.use_bulk_tracking:

        #    self.unrecorded_hits = []

    @classmethod

    def launch(cls, recorder_count):

        """

        Launch a bunch of Recorder objects in a separate thread.

        """

        for i in range(recorder_count):            

            recorder = Recorder()

            cls.recorders.append(recorder)

            #run = recorder._run_bulk if config.options.use_bulk_tracking else recorder._run_single

            run = recorder._run_bulk

            t = threading.Thread(target=run)

            t.daemon = True

            t.start()

            logging.debug('Launched recorder')

    @classmethod

    def add_hits(cls, all_hits):

        """

        Add a set of hits to the recorders queue.

        """

        # Organize hits so that one client IP will always use the same queue.

        # We have to do this so visits from the same IP will be added in the right order.

        hits_by_client = [[] for r in cls.recorders]

        for hit in all_hits:

            hits_by_client[hit.get_visitor_id_hash() % len(cls.recorders)].append(hit)

        for i, recorder in enumerate(cls.recorders):

            recorder.queue.put(hits_by_client[i])

    @classmethod

    def wait_empty(cls):

        """

        Wait until all recorders have an empty queue.

        """

        for recorder in cls.recorders:

            recorder._wait_empty()

    def _run_bulk(self):

        while True:

            try:

                hits = self.queue.get()

            except:

                # TODO: we should log something here, however when this happens, logging.etc will throw

                return

            if len(hits) > 0:

                try:

                    self._record_hits(hits)

                except Matomo.Error as e:

                    fatal_error(e, hits[0].filename, hits[0].lineno) # approximate location of error

            self.queue.task_done()

    def _run_single(self):

        while True:

            if len(self.unrecorded_hits) > 0:

                hit = self.unrecorded_hits.pop(0)

                try:

                    self._record_hits([hit])

                except Matomo.Error as e:

                    fatal_error(e, hit.filename, hit.lineno)

            else:

                self.unrecorded_hits = self.queue.get()

                self.queue.task_done()

    def _wait_empty(self):

        """

        Wait until the queue is empty.

        """

        while True:

            if self.queue.empty():

                # We still have to wait for the last queue item being processed

                # (queue.empty() returns True before queue.task_done() is

                # called).

                self.queue.join()

                return

            time.sleep(1)

    def date_to_matomo(self, date):

        date, time = date.isoformat(sep=' ').split()

        return '%s %s' % (date, time.replace('-', ':'))

    def _get_hit_args(self, hit):

        """

        Returns the args used in tracking a hit, without the token_auth.

        """

        #site_id, main_url = resolver.resolve(hit)

        site_id = config.options['Matomo_Parameters']['idSite']

        #repositoy base url

        main_url = config.options['Matomo_Parameters']['repository_base_url']

        #stats.dates_recorded.add(hit.date.date())

        path = hit.path

        '''

        query_string_delimiter="?"

        if hit.query_string:

            path += config.options.query_string_delimiter + hit.query_string

        '''

        # only prepend main url / host if it's a path

        url_prefix = self._get_host_with_protocol(hit.host, main_url) if hasattr(hit, 'host') else main_url

        url = (url_prefix if path.startswith('/') else '') + path[:1024]

        # handle custom variables before generating args dict

        #if hit.is_robot:

        #    hit.add_visit_custom_var("Bot", hit.user_agent)

        #else:

        #    hit.add_visit_custom_var("Not-Bot", hit.user_agent)

        if (hit.referrer.find("?") >=0):

            hit.referrer = hit.referrer.split("?")[0]+" "

        args = {

            'rec': '1',

            'apiv': '1',

            'url': url,

            'urlref': hit.referrer[:1024],

            'cip': hit.ip,

            'cdt': self.date_to_matomo(hit.date),

            'idsite': site_id,

            'ua': hit.user_agent

        }

        # idsite is already determined by resolver

        if 'idsite' in hit.args:

            del hit.args['idsite']

        args.update(hit.args)

        if hit.is_download:

            args['download'] = args['url']

        #if config.options.enable_bots:

        args['bots'] = '1'

        '''

        if hit.is_error or hit.is_redirect:

            args['action_name'] = '%s%sURL = %s%s' % (

                hit.status, '/',

                urllib.quote(args['url'], ''),

                ("%sFrom = %s" % (

                    '/',

                    urllib.quote(args['urlref'], '')

                ) if args['urlref'] != ''  else '')

            )

        '''

        if hit.generation_time_milli > 0:

            args['gt_ms'] = int(hit.generation_time_milli)

        if hit.event_category and hit.event_action:

            args['e_c'] = hit.event_category

            args['e_a'] = hit.event_action

            if hit.event_name:

                args['e_n'] = hit.event_name

        if hit.length:

            args['bw_bytes'] = hit.length

        # convert custom variable args to JSON

        if 'cvar' in args and not isinstance(args['cvar'], str):

            args['cvar'] = json.dumps(args['cvar'])

        if '_cvar' in args and not isinstance(args['_cvar'], str):

            args['_cvar'] = json.dumps(args['_cvar'])