Versions Compared

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

...

Code Block
$ sudo apt-get install python-setuptools git libpq-dev postgresql python-dev postfix nginx-full uwsgi uwsgi-plugin-python libffi-dev
$ sudo easy_install pip
$ sudo pip install virtualenv
$ sudo adduser --disabled-password --home /var/opt/norduni ni

...

Download neo4j-community from http://neo4j.com/download/. NORDUnet and SUNET run 2.1.8. 2.3.2 has been tested and did not work as expected.

Code Block
$ tar xvfz neo4j-community-2.1.78-unix.tar.gz
$ sudo mv neo4j-community-2.1.78 /var/opt/.
$ sudo ln -s /var/opt/neo4j-community-2.1.78 /var/opt/neo4j-community
$ cd /var/opt/neo4j-community
$ sudo ./bin/neo4j-installer install

...

Code Block
$ . norduni_environment/bin/activate
$ pip install -r norduni/requirements/prod.txt

Django settings

Change the django settings.

Code Block
$ cd norduni/src/niweb/niweb/
$ cp generic_settings.pydotenv settings.pyenv
$ vi settings.pyenv

Change at least the database settingsThe following settings need to be changed.

Code Block
REPORTS_TO=
DB_PASSWORD=
DEFAULT_FROM_EMAIL=
EMAIL_HOST=
SECRET_KEY=

Check if your settings are ok.

Code Block
$ python manage.py syncdb
$ python manage.py migrate apps.noclook
$ python manage.py migrate actstream
$ python manage.py migrate tastypie
$ python manage.py collectstatic
$ python manage.py runserver

Now you should be able connect to the machine with your browser on http://localhost:8000 and see the NOCLook app index page.

Deploying NOCLook

uwsgi

Create a uwsgi configuration file.

Code Block
$ sudo vi /etc/uwsgi/apps-available/noclook.ini

The following configuration should be a good start.

[uwsgi]
# Django-related settings
plugins = python
protocol = uwsgi
# the base directory (full path)
chdir# Database settings
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'norduni',
        'USER': 'postgres',
        'PASSWORD': 'secret',
        'HOST': 'localhost'
    }
}

Check if your settings are ok.

Code Block
$ python manage.py syncdb
$ python manage.py migrate apps.noclook
$ python manage.py migrate actstream
$ python migrate tastypie
$ python manage.py runserver

Now you should be able connect to the machine with your browser on http://localhost:8000 and see the NOCLook app index page.

Deploying NOCLook

uwsgi

Create a uwsgi configuration file.

Code Block
$ sudo vi /etc/uwsgi/apps-available/noclook.ini

The following configuration should be a good start.

[uwsgi]
# Django-related settings
plugins = python
protocol = uwsgi
# the base directory (full path)
chdir = /var/opt/norduni/norduni/src/niweb/
# Django's wsgi file
wsgi-file       = /var/opt/norduni/norduni/src/niweb/niweb/wsgi.py
env             = DJANGO_SETTINGS_MODULE=niweb.settings.prod
# the virtualenv (full path)
home            = /var/opt/norduni/norduni_environment
# logging
daemonize          = /var/optlog/norduniuwsgi/norduni/src/niweb/app/noclook.log
# Django's wsgi file
wsgi-fileprocess-related settings
# master
master          = /var/opt/norduni/norduni/src/niweb/niweb/wsgi.py
envtrue
# maximum number of worker processes
processes       = 5
#threads        = DJANGO_SETTINGS_MODULE=niweb.settings 2
max-requests    = 5000
# the virtualenvsocket (use the full path)
home to be safe
socket          = /var/opt/norduni/norduni_environment127.0.0.1:8001
# logging
daemonizeclear environment      = /var/log/uwsgi/app/noclook.log
# process-related settings
# master
masteron exit
vacuum          = true
# maximum number of worker processes
processes       = 5
#threads        = 2
max-requests    = 5000
# the socket (use the full path to be safe
socket          =

Link the configuration in to the correct directory.

Code Block
sudo ln -s /etc/uwsgi/apps-available/noclook.ini /etc/uwsgi/apps-enabled/noclook.ini

Make temp dir and log dir writable by the uwsgi user (www-data on ubuntu)

Code Block
sudo chown -R ni:www-data /tmp/django_cache
sudo chmod -R g+w /tmp/django_cache
 
sudo chown -R ni:www-data /var/opt/norduni/norduni/src/niweb/logs/
sudo chmod -R g+w /var/opt/norduni/norduni/src/niweb/logs/

 

nginx

Setup new dhparam file 2048 should suffice, but if you like you can go with 4096 instead:

Code Block
$ sudo openssl dhparam -out /etc/ssl/dhparams.pem 2048

Configure nginx.

Code Block
title/etc/nginx/sites-available/default
$ sudo vi /etc/nginx/sites-available/default

The following configuration should be a good start.

upstream django {    
    server 127.0.0.1:8001
; # clearfor a environmentweb onport exit
vacuum  socket
}

server {
    listen    = true

Link the configuration in to the correct directory.

Code Block
sudo ln -s /etc/uwsgi/apps-available/noclook.ini /etc/uwsgi/apps-enabled/noclook.ini
nginx

Configure nginx.

Code Block
title/etc/nginx/sites-available/default
$ sudo vi /etc/nginx/sites-available/default

The following configuration should be a good start.

upstream django {    
    server 127.0.0.1:8001; # for a web port socket     80;
    listen         [::]:80;
    server_name    ni.nordu.net;
    rewrite        ^ https://$server_name$request_uri? permanent;
}

server {
    listen listen443;
         80;
    listen  listen [::]:443 default ipv6only=on; ## listen for ipv6
    ssl   [::]:80on;
	ssl_certificate /etc/ssl/ni_nordu_net.crt;
    serverssl_certificate_name    ni.nordu.net;key /etc/ssl/ni_nordu_net.key;

    rewrite # https://cipherli.st
    ssl_prefer_server_ciphers on;
  ^ https://$server_name$request_uri? permanent;
}

server {
    listen 443  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
    ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
    listen [::]:443 default ipv6only=on; ## listen for ipv6ssl_session_cache shared:SSL:10m;
    ssl_ecdh_curve secp384r1;
    ssl on;
	ssl_certificate _dhparam /etc/ssl/ni_nordu_net.crt;dhparams.pem;
    
    sslserver_certificate_key /etc/ssl/ni_nordu_net.keyname ni.nordu.net;

    # PFS settings from http://people.adams.edu/~cdmiller/posts/NSA_SSL_settings_for_nginx_and_apache/location /static/ {
    # NOTE: these settings alias exclude Win XP    with IE 6 /var/opt/norduni/norduni/src/niweb/niweb/static/;
    #
    autoindex    on;
     ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
   access_log   off;
        expires     ssl_prefer_server_ciphers on; 30d; 
    ssl_ciphers DHE-RSA-CAMELLIA256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:DES-CBC3-SHA:!ADH:!aNULL;}

    location / {
    
    include     server_name ni.nordu.net;

/etc/nginx/uwsgi_params;
     location /static/ {
  proxy_set_header   Host     root         /var/opt/norduni/norduni/src/niweb/sitestatic/    $host;
        autoindexproxy_set_header    on;
X-Real-IP         access_log   off$remote_addr;
        expiresproxy_set_header   X-Forwarded-For   30d; 
    }

$proxy_add_x_forwarded_for;
    location / {
  proxy_set_header   X-Forwarded-Proto   include     /etc/nginx/uwsgi_params $scheme;
        proxy_set_header  proxy_redirect  Host   off;
        uwsgi_pass      $hostdjango;
    }    proxy_set_header   X-Real-IP            $remote_addr;
        proxy_set_header   X-Forwarded-For      $proxy_add_x_forwarded_for;
        proxy_set_header   X-Forwarded-Proto    $scheme;
        proxy_redirect     off;
        uwsgi_pass  django;
    }   
}
SAML SP

If you want to set up NOCLook as a SAML SP you need to install the following packages and Python modules.

Code Block
$ sudo apt-get install libffi-dev xmlsec1
$ sudo -u ni -i
$ . norduni_environment/bin/activate
$ pip install djangosaml2

}
SAML SP

If you want to set up NOCLook as a SAML SP you need to install the following packages and Python modules.

Code Block
$ sudo apt-get install libffi-dev xmlsec1
$ sudo -u ni -i
$ . norduni_environment/bin/activate
$ pip install djangosaml2

You then need to uncomment the lines in settings.py that imports and sets up djangosaml2. You also have to create a pysaml2 configuration.
All this is best described in the documentation at https://pypi.python.org/pypi/djangosaml2.

Local saml metadata

To speed up login you can use local metadata. This metadata still needs to be updated and verified, and for that you can use https://github.com/NORDUnet/metadata-updater 

You need to configure djangosaml2 to use local metadata, and you will have to add the meta-dataupdater to cron, preferably by running crontab -e as the ni user. Once an hour is reasonable, once a day can be ok, once a week might be tiresome when the cert expiresYou then need to uncomment the lines in settings.py that imports and sets up djangosaml2. You also have to create a pysaml2 configuration.
All this is best described in the documentation at https://pypi.python.org/pypi/djangosaml2.

Collecting and processing network data

...

Code Block
titleBlank output from NORDUnet site documentation example
{
    "host": {
        "csv_producer": {
            "address": "", 
            "area": "", 
            "city": "", 
            "comment": "", 
            "country": "", 
            "floor": "", 
            "latitude": "", 
            "longitude": "", 
            "meta_type": "", 
            "name": "", 
            "node_type": "", 
            "owner_id": "", 
            "postcode": "", 
            "responsible_for": "", 
            "room": "", 
            "site_type": "", 
            "telenor_subscription_id": ""
        }, 
        "name": "", 
        "version": 1
    }
}

The consumer script should only be run once as it does not update the sites, only creates new.

The JSON file directory is then inserted in to the database using noclook_site_csv_consumer.py.

Change the path at the top of the script to be able to import norduni_client.py.

Then run:

Code Block
python noclook_site_csv_consumer.py -D /path/to/site_files/json

Daily database update

1
    }
}

The consumer script should only be run once as it does not update the sites, only creates new.

The JSON file directory is then inserted in to the database using noclook_site_csv_consumer.pyThe producers are run with a cron job and the script noclook_consumer.py is used to run the three inserting/updating scripts (noclook_juniper_consumer.py, noclook_alcatel_consumer.py and noclook_nmap_consumer.py).

Change the path at the top of the script to be able to import norduni_client.py.

...

to be able to import norduni_client.py.

Then run:

Code Block
python noclook_site_csv_consumer.py -C template.conf -I

Setting up a local/development NOCLook

D /path/to/site_files/json

Daily database update

The producers are run with a cron job and the script noclook_consumer.py is used to run the three inserting/updating scripts (noclook_juniper_consumer.py, noclook_alcatel_consumer.py and noclook_nmap_consumer.py).

Change the path at the top of the script to be able to import norduni_client.py.

Code Block
[data]
juniper_conf = /path/to/juniper/json
nmap_services = /path/to/nmap/json
alcatel_isis = /path/to/alcate/json
noclook = #Used for loading backup.

Then run:

Code Block
python noclook_consumer.py -C template.conf -I

Setting up a local/development NOCLook

Code Block
# Clone a convenience repo
$ git clone https://github.com/NORDUnet/norduni-developer
$ cd norduni-developer
# Start dependencies
$ ./start.sh

# Clone NOCLook project repo
$ git clone https://git.nordu.net/norduni.git
$ cd norduni
Code Block
$ git clone https://git.nordu.net/norduni.git
# Download neo4j docker image and start it
$ docker pull tpires/neo4j
$ docker run -d -v /path_to_repo/norduni/docker/neo4j.properties:/var/lib/neo4j/conf/neo4j.properties -v /opt/docker/neo4jdata:/var/lib/neo4j/data -p 7474:7474 tpires/neo4j
# Create the indexes with curl
$ curl -D - -H "Content-Type: application/json" --data '{"name" : "node_auto_index","config" : {"type" : "fulltext","provider" : "lucene"}}' -X POST http://localhost:7474/db/data/index/node/
$ curl -D - -H "Content-Type: application/json" --data '{"name" : "relationship_auto_index","config" : {"type" : "fulltext","provider" : "lucene"}}' -X POST http://localhost:7474/db/data/index/relationship/
# Create a virtualenv and activate it
$ virtualenv env
$ . env/bin/activate
# Install the python packages
$ pip install paver
$ pip install -r /path_to_repo/requirementsrequirements/dev.txt
# Create a settings.py from the template /path_to/repo/src/niweb/niweb file
$ cp /path_to_repo/src/niweb/niweb/generic_settings.py /path_to/repo/src/niweb/dotenv src/niweb/settings.pydevenv
# Sync the db
$ python /path_to_repo/src/niweb/manage.py syncdb
$ python /path_to_repo/src/niweb/manage.py migrate apps.noclook
# Run the app
$ python /path_to_repo/src/niweb/manage.py migrate actstream
$ python /path_to_repo/src/niweb/manage.py migrate tastypie runserver

Upgrading to newest versions

This is the general procedure for upgrading to newest version of norduni.

Code Block
# stash current local changes and update
$ git stash
$ git pull origin master
$ git stash apply
 
# Run the appmigrations
$ python /path_to_repo/src/niweb/manage.py runservermigrate
  
# OptionalPip postgres instead of sqlite3, don't forget to change database settings in settings.py.
# Download postgres docker image and start it
$ docker pull orchardup/postgresql
$ docker run -d -p 5432:5432 -e POSTGRESQL_USER=ni -e POSTGRESQL_PASS=docker -e POSTGRESQL_DB=norduni -v /opt/docker/postgresql_data/:/var/lib/postgresql/ orchardup/postgresql
update requirements
$ pip install -U -r requirements/prod.txt
 
# Collect statics
$ python /path_to_repo/src/niweb/manage.py collectstatic
 
# Restart uwsgi
$ sudo services uwsgi restart