<repo_name>dwxero/Immoscout<file_sep>/src/main/java/com/dwxero/crawl/immoscout/repository/PropertyRepository.java
package com.dwxero.crawl.immoscout.repository;

import com.dwxero.crawl.immoscout.entity.Property;
import com.dwxero.crawl.immoscout.entity.PropertyState;
import org.springframework.data.domain.Pageable;
import org.springframework.data.mongodb.repository.MongoRepository;

import java.util.List;

public interface PropertyRepository extends MongoRepository<Property, String> {

    Property findById(String id);

    Property findByUrl(String url);

    Property findByExposeId(String exposeId);

    List<Property> findByState(PropertyState state);

    List<Property> findByState(PropertyState state, Pageable pageable);
}
<file_sep>/readme.md
# start mongo db
sudo  ./dev/db/mongodb-osx-x86_64-2.6.5/bin/mongod

# connect to mongo db via terminal 
./dev/db/mongodb-osx-x86_64-2.6.5/bin/mongo

# useful commands
show dbs; 
show collections;
db.property.find().pretty()
db.property.remove({})<file_sep>/src/test/java/com/dwxero/crawl/immoscout/helper/DataExtractorHelperTest.java
package com.dwxero.crawl.immoscout.helper;

import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Test;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

public class DataExtractorHelperTest {

    private String html;

    @Before
    public void loadExampleHtmlFile() throws Exception {
        this.html = IOUtils.toString(this.getClass().getResourceAsStream("exampleRawHtml.html"),
                "UTF-8"
        );
    }

    @Test
    public void testThatTitleExtractionWorks() {
        // given
        String title;
        DataExtractorHelper helper = new DataExtractorHelper();

        // when
        title = helper.getTitleFromHtml(html);

        // then
        assertNotNull(title);
        assertTrue("Wohntraum auf großem Eckgrundstück im noblen Junkersdorf".equals(title));
    }

    @Test
    public void testThatAddressExtractionWorks() {
        // given
        String address;
        DataExtractorHelper helper = new DataExtractorHelper();

        // when
        address = helper.getAddressFromHtml(html);

        // then
        assertNotNull(address);
        assertTrue("Amselstraße 19, 50858 Köln, Junkersdorf".equals(address));
    }

    @Test
    public void testThatPriceExtractionWorks() {
        // given
        double price;
        DataExtractorHelper helper = new DataExtractorHelper();

        // when
        price = helper.getPriceFromHtml(html);

        // then
        assertEquals(1275000.00d, price, 0.001d);
    }

    @Test
    public void testThatRoomsExtractionWorks() {
        // given
        double rooms;
        DataExtractorHelper helper = new DataExtractorHelper();

        // when
        rooms = helper.getRoomsFromHtml(html);

        // then
        assertEquals(5.0d, rooms, 0.01d);
    }

    @Test
    public void testThatBedroomsExtractionWorks() {
        // given
        double bedrooms;
        DataExtractorHelper helper = new DataExtractorHelper();

        // when
        bedrooms = helper.getBedroomsFromHtml(html);

        // then
        assertEquals(4.0d, bedrooms, 0.01d);
    }

    @Test
    public void testThatBathroomsExtractionWorks() {
        // given
        double bathrooms;
        DataExtractorHelper helper = new DataExtractorHelper();

        // when
        bathrooms = helper.getBathroomsFromHtml(html);

        // then
        assertEquals(1.0d, bathrooms, 0.01d);
    }

    @Test
    public void testThatSquareMetersLivingExtractionWorks() {
        // given
        double squareMetersLiving;
        DataExtractorHelper helper = new DataExtractorHelper();

        // when
        squareMetersLiving = helper.getSquareMetersLivingFromHtml(html);

        // then
        assertEquals(172.0d, squareMetersLiving, 0.001d);
    }


    @Test
    public void testThatSquareMetersLandExtractionWorks() {
        // given
        double squareMetersLand;
        DataExtractorHelper helper = new DataExtractorHelper();

        // when
        squareMetersLand = helper.getSquareMetersLandFromHtml(html);

        // then
        assertEquals(539.0d, squareMetersLand, 0.001d);
    }

    @Test
    public void testThatCommissionNoteExtractionWorks() {
        // given
        String commissionNote;
        DataExtractorHelper helper = new DataExtractorHelper();

        // when
        commissionNote = helper.getCommissionNoteFromHtml(html);

        // then
        assertTrue("Nein".equals(commissionNote));
    }
}<file_sep>/src/main/java/com/dwxero/crawl/immoscout/DetailsCrawler.java
package com.dwxero.crawl.immoscout;

import com.dwxero.crawl.immoscout.entity.Property;
import com.dwxero.crawl.immoscout.entity.PropertyState;
import com.dwxero.crawl.immoscout.service.PropertyService;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.util.List;

/**
 * this class crawls the detail information pages for the given properties in the database.
 * the crawler only stores the raw html code in the property.
 */
@Component
public class DetailsCrawler implements Crawler {

    private static final Logger LOG = LoggerFactory.getLogger(ListCreationCrawler.class);

    private PropertyService propertyService;

    @Autowired
    public DetailsCrawler(PropertyService propertyService) {
        this.propertyService = propertyService;
    }

    @Override
    @Scheduled(fixedDelay = 10 * 1000)
    public void crawl() {
        LOG.info("------------------------------------");
        LOG.info("Start of DetailsCrawler");
        LOG.info("------------------------------------");

        List<Property> listOfProperties = propertyService.getNewProperties();
        for (Property property : listOfProperties) {
            property = getInformationFromDetailsPage(property);
            propertyService.updateProperty(property);
        }

        LOG.info("------------------------------------");
        LOG.info("Finish of DetailsCrawler");
        LOG.info("------------------------------------");
    }

    private Property getInformationFromDetailsPage(Property property) {
        try {
            LOG.info("Fetching information for property:" + property);
            Connection.Response response = Jsoup.connect(property.getUrl()).method(Connection.Method.GET).execute();
            if (response.statusCode() == 200) {
                Document detailsPage = response.parse();
                property.setRawHtml(detailsPage.html());
                property.setState(PropertyState.DOWNLOADED);
                return property;
            } else {
                property.setState(PropertyState.OFFLINE);
                return property;
            }
        } catch (IOException e) {
            LOG.error(e.getMessage());
            return property;
        }
    }
}
