<file_sep>rootProject.name = 'serveranalytics'

<file_sep>gradle --daemon clean test distZip -x processResources
<file_sep>package de.tlongo.serveranalytics.test;

import org.hibernate.annotations.Type;

import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import java.time.LocalDateTime;

/**
 * Created by <NAME> on 10/9/14.
 */
@Entity(name = "foo")
public class Foo {
    @Type(type="de.tlongo.serveranalytics.services.logfileservice.LocalDateTimeUserType")
    public LocalDateTime date;

    @Id
    @GeneratedValue(strategy = GenerationType.AUTO)
    long id;

    public Foo() {

    }

    public Foo(LocalDateTime date) {
        this.date = date;
    }
}
<file_sep>package de.tlongo.serveranalytics.services.logfileservice;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by tomas on 18.09.14.
 */
public class LogFileParser {
    static Logger logger = LoggerFactory.getLogger(LogFileParser.class);
    static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z");
    private static final List<LogEntry> EMPTY_LIST = new ArrayList<>();

    public static boolean isEntryValid(LogEntry entry) {
        return entry.getStatus() != -9999;
    }

    /**
     * Parses a nginx log file line by line, extracts information and returns a list
     * of LogEntry objects
     *
     * @param logFile The log file to parse
     *
     * @return A list of LogEntry objects. An empty list if an error occurs.
     */
    public static List<LogEntry> parseLogFile(File logFile, String producingApp) {
        try {
            List<LogEntry> entryList = new ArrayList<>();
            BufferedReader reader = new BufferedReader(new FileReader(logFile));
            String line = null;
            while ((line = reader.readLine()) != null) {
                if (line.isEmpty()) continue;

                LogEntry entry = new LogEntry();
                String[] tokens = line.split("@");

                if (tokens.length != 5) {
                    // Trying to split this log line into its components by the delimeter '@'
                    // caused an error because '@' was part of the line itself.
                    // Mark entry as invalid in order to get some metrics on how often this
                    // occurs. We might have to change the separator.
                    entry.setStatus(-9999);
                } else {
                    entry.setAddress(tokens[0]);
                    entry.setDate(Timestamp.valueOf(LocalDateTime.parse(tokens[1], formatter)));
                    entry.setStatus(Integer.parseInt(tokens[3]));
                    entry.setAgent(tokens[4]);
                    entry.setProducedBy(producingApp);

                    String[] requestTokens = splitRequestString(tokens[2]);
                    if (requestTokens.length != 3) {
                        entry.setStatus(-9999);
                    } else {
                        entry.setRequestMethod(requestTokens[0]);
                        entry.setRequestUri(requestTokens[1]);
                        entry.setRequestProtocol(requestTokens[2]);
                    }
                }

                entryList.add(entry);
            }
            return entryList;
        } catch (FileNotFoundException e) {
            logger.error("Could not find file under path {}", logFile.getAbsolutePath());
            return EMPTY_LIST;
        } catch (IOException e) {
            logger.error("Error parsing logfile {}\n{}", logFile.getAbsolutePath(), e);
            return EMPTY_LIST;
        }
    }

    private static String[] splitRequestString(String requestString) {
        // tokens = [method, uri, protocol]
        String[] tokens = requestString.split(" ");



        return tokens;
    }

    /**
     * Parses log files located in the specified directory.
     *
     * @return A List of log entries extracted from the files in the directory.
     */
    public static List<LogEntry> parseLogDirectory(String logDir) {
        File dir = new File(logDir);
        if (!dir.isDirectory()) {
            logger.error("{} is not a directory", logDir);
            return EMPTY_LIST;
        }

        logger.info("Parsing log files in directory {}", dir.getAbsolutePath());
        try {
            List<LogEntry> logEntryList = new ArrayList<>();
            DirectoryStream<Path> directoryStream = Files.newDirectoryStream(dir.toPath());
            directoryStream.forEach(path -> {
                File logFile = new File(path.toAbsolutePath().toString());
                if (!logFile.isDirectory() && logFile.getName().contains("log")) {
                    logger.info("Parsing logfile {}", logFile.getAbsolutePath());
                    List<LogEntry> list = parseLogFile(logFile, dir.toPath().getFileName().toString());

                    logEntryList.addAll(list);
                }
            });

            return logEntryList;
        } catch (IOException e) {
            logger.error("Could not find directory under {}", logDir);
            return EMPTY_LIST;
        }
    }
}
<file_sep>apply plugin: 'java'
apply plugin: 'application'

sourceCompatibility = 1.8
version = '2.0.3'

mainClassName = "de.tlongo.serveranalytics.services.logfileservice.LogService"

repositories {
    mavenCentral()
}

dependencies {
    compile 'com.sparkjava:spark-core:2.0.0'
    compile 'org.codehaus.groovy:groovy-all:2.3.6'
    compile 'commons-cli:commons-cli:1.2'
    compile 'com.google.code.gson:gson:2.2.4'
    compile 'org.springframework.data:spring-data-jpa:1.7.0.RELEASE'
    compile 'ch.qos.logback:logback-classic:1.1.1'
    compile 'org.hibernate.javax.persistence:hibernate-jpa-2.0-api:1.0.1.Final'
    compile('org.hibernate:hibernate-entitymanager:4.3.6.Final') {
        //The pacage that comes with hibernate is not Java8 compatible
        exclude group: 'org.javassist'
    }
    compile 'org.javassist:javassist:3.18.2-GA'
    compile 'mysql:mysql-connector-java:5.1.6'
    compile 'com.rabbitmq:amqp-client:3.4.4'

    testCompile 'org.springframework:spring-test:4.0.7.RELEASE'
    testCompile 'org.hamcrest:hamcrest-all:1.3'
    testCompile group: 'junit', name: 'junit', version: '4.11'
}

installApp {
    // copy the configs to external folder
    into("lib/config") {
        from 'src/main/resources'
        include '*'
    }
}

distZip {
    archiveName = "${project.name}.zip"
    ext.versionFileName ="${version}.version"
    ext.versionFile = file(versionFileName)

    // copy the configs to external folder
    into("$baseName/lib/config") {
        from 'src/main/resources'
        include '*'
    }

    into("$baseName") {
        from "${project.projectDir.toPath().toAbsolutePath().toString()}"
        include versionFileName
    }

    doFirst {
        versionFile.createNewFile()
    }

    doLast {
        versionFile.delete()
    }
}

// Adde the config folder to the classpath of the config script
startScripts.classpath += files("config")
<file_sep>package de.tlongo.serveranalytics.test;

import de.tlongo.serveranalytics.services.logfileservice.LogEntry;
import de.tlongo.serveranalytics.services.logfileservice.LogEntryRepository;
import de.tlongo.serveranalytics.services.logfileservice.LogFileParser;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import java.io.File;
import java.net.URL;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.Month;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collector;
import java.util.stream.Collectors;

import static org.hamcrest.Matchers.*;
import static org.hamcrest.MatcherAssert.*;

/**
 * Created by tomas on 19.09.14.
 */
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration("/springdataconfig.xml")
public class TestPersistence {

    @Autowired
    LogEntryRepository repo;

    @Before
    public void clearDatabase() {
        repo.deleteAll();
    }

    @Test
    @Ignore
    public void testPersistLogEntry() throws Exception {
        LocalDateTime currentDateTime = LocalDateTime.now().truncatedTo(ChronoUnit.SECONDS);
        LogEntry entry = new LogEntry();
        entry.setAgent("Mausi");
        entry.setDate(Timestamp.valueOf(currentDateTime));
        repo.save(entry);

        LogEntry loaded = repo.findOne(entry.getId());
        assertThat(loaded, notNullValue());
        assertThat(loaded.getAgent(), equalTo("Mausi"));
        assertThat(loaded.getDate(), equalTo(Timestamp.valueOf(currentDateTime)));

        System.out.println(loaded.getDate().toString());
    }

    @Test
    @Ignore
    public void testPersistParsedFile() throws Exception {
        List<LogEntry> logEntries = LogFileParser.parseLogFile(new File("src/test/logdir/fixed/access.log.2"), "Test");
        logEntries.forEach(entry -> {
            repo.save(entry);
        });

        long count = repo.count();
        assertThat(count, equalTo((long)logEntries.size()));


    }

    @Test
    @Ignore
    public void testPersistParsedDirectory() throws Exception {
        List<LogEntry> logEntries = LogFileParser.parseLogDirectory("src/test/logdir/fixed");
        logEntries.forEach(entry -> {
            repo.save(entry);
        });

        long count = repo.count();
        assertThat(count, equalTo((long) logEntries.size()));
    }

    @Test
    public void testFindByDateRange() throws Exception {
        String dir = new ClassPathResource("/logdir/fixed/access.log.2").getFile().toPath().normalize().toAbsolutePath().toString();
        List<LogEntry> logEntries = LogFileParser.parseLogFile(new File(dir), "test");
        logEntries.forEach(entry -> {
            repo.save(entry);
        });

        long count = repo.count();
        assertThat(count, equalTo((long) logEntries.size()));

        Timestamp start = Timestamp.valueOf(LocalDateTime.of(2014, Month.OCTOBER, 18, 19, 57, 2).truncatedTo(ChronoUnit.DAYS));
        Timestamp end = Timestamp.valueOf(LocalDateTime.of(2014, Month.OCTOBER, 19, 19, 57, 2).truncatedTo(ChronoUnit.DAYS));
        List<LogEntry> entries = repo.findByDateRange(start, end);
        assertThat(entries, notNullValue());
        assertThat(entries, hasSize(6));


        start = Timestamp.valueOf(LocalDateTime.of(2014, Month.SEPTEMBER, 22, 19, 57, 2).truncatedTo(ChronoUnit.DAYS));
        end = Timestamp.valueOf(LocalDateTime.of(2014, Month.SEPTEMBER, 23, 19, 57, 2).truncatedTo(ChronoUnit.DAYS));
        entries = repo.findByDateRange(start, end);
        assertThat(entries, notNullValue());
        assertThat(entries, hasSize(73));
    }

    @Test
    public void testFindByIdList() throws Exception {
        String logfile = new ClassPathResource("/logdir/fixed/access.log.3").getFile().toPath().toAbsolutePath().normalize().toString();
        List<LogEntry> logEntries = LogFileParser.parseLogFile(new File(logfile), "test");
        repo.save(logEntries);

        String idList = logEntries.stream().
                            map(entry -> String.valueOf(entry.getId())).
                            collect(Collectors.joining(","));

        System.out.println("<<<" + idList + ">>>");

        List<Long> longIds = Arrays.stream(idList.split(",")).map(stringid -> Long.parseLong(stringid)).collect(Collectors.toCollection(ArrayList::new));

        idList = longIds.stream().
                map(longId -> longId.toString()).
                collect(Collectors.joining(","));

        System.out.println("<<<" + idList + ">>>");

        long now = System.currentTimeMillis();
        List<LogEntry> finalFetch = repo.findAll(longIds);
        double time = (double)(System.currentTimeMillis() - now) / 1000.0;
        System.out.println("TIME: " + time);

        assertThat(finalFetch, hasSize(logEntries.size()));
        assertThat(finalFetch, equalTo(logEntries));
    }
}
<file_sep>#Taks

##Elastic Search Integration

ElasticSearch soll Logeinträge indzieren. Hauptgrund hierfür ist die spätere  Verwendung von Kibana zur Visualisierung.

Der Logfileservice soll nach jedem Parselauf die aktuell gefundenen Einträge in eine Queue schreiben (zusätzlich
zum Eintragen in eine DB). Dort horcht jeder mit, der an den Logeinträgen interessiert ist.

* GET Route, die eine Liste von IDs entgegennimmt und die entsprechenden Logs aus der DB liefert
* Nach jedem täglichen Import, die frischen LogIDs in die Queue schreiben<file_sep>logfileservice.logdir=src/test/logdir/fixed

logfileservice.persisting.perLogFile=true<file_sep>package de.tlongo.serveranalytics.test;

import org.springframework.data.jpa.repository.JpaRepository;

/**
 * Created by tomas on 10/9/14.
 */
public interface FooRepo extends JpaRepository<Foo, Long> {
}
