crawl

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

BASEDIR=$(dirname "$0")
SEED_DIR="$BASEDIR/seed/urls.txt"
NUTCH_BIN="$BASEDIR/build/apache-nutch-2.3.1/runtime/local/bin/nutch"
 
function __bin_nutch {
    # run $bin/nutch, exit if exit value indicates error

    echo "$NUTCH_BIN $@" ;# echo command and arguments
    "$NUTCH_BIN" "$@"

    RETCODE=$?
    if [ $RETCODE -ne 0 ]
    then
        echo "Error running:"
        echo "$NUTCH_BIN $@"
        echo "Failed with exit value $RETCODE."
        exit $RETCODE
    fi
}

###### Actual Crawling and Indexing procedure ##########
# Note: On the first run, this will only crawl the injected URLs. The procedure above is supposed to be repeated regularly to keep the index up to date.

# inject them into Nutch by giving a file URL
echo "Injecting urls from $SEED_DIR"
__bin_nutch inject $SEED_DIR

# Generate a new set of URLs to fetch. topN parameter decides how many pages nutch should crawl per depth. If you estimate a website to have 3000 pages then you can specify a depth value of 3 and a topN value of 1000 for a successful crawl of 3000 documents.
echo "Generate urls: "
__bin_nutch generate -topN 5

echo "Fetch urls: "
__bin_nutch fetch -all

echo "parsing fetched pages: "
__bin_nutch parse -all

echo "Update Nutch's internal database: "
__bin_nutch updatedb -all

echo "Index documents in Discovery: "
__bin_nutch index -all