# # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host: # Used: http://example.com/robots.txt # Ignored: http://example.com/site/robots.txt # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/robotstxt.html User-agent: * # CSS, JS, Images Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /profiles/*.css$ Allow: /profiles/*.css? Allow: /profiles/*.js$ Allow: /profiles/*.js? Allow: /profiles/*.gif Allow: /profiles/*.jpg Allow: /profiles/*.jpeg Allow: /profiles/*.png Allow: /profiles/*.svg # Directories Disallow: /core/ Disallow: /profiles/ # Files Disallow: /README.md Disallow: /composer/Metapackage/README.txt Disallow: /composer/Plugin/ProjectMessage/README.md Disallow: /composer/Plugin/Scaffold/README.md Disallow: /composer/Plugin/VendorHardening/README.txt Disallow: /composer/Template/README.txt Disallow: /modules/README.txt Disallow: /sites/README.txt Disallow: /themes/README.txt Disallow: /web.config # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /filter/tips Disallow: /node/add/ Disallow: /search/ Disallow: /user/register Disallow: /user/password Disallow: /user/login Disallow: /user/logout Disallow: /media/oembed Disallow: /*/media/oembed # Paths (no clean URLs) Disallow: /index.php/admin/ Disallow: /index.php/comment/reply/ Disallow: /index.php/filter/tips Disallow: /index.php/node/add/ Disallow: /index.php/search/ Disallow: /index.php/user/password Disallow: /index.php/user/register Disallow: /index.php/user/login Disallow: /index.php/user/logout Disallow: /index.php/media/oembed Disallow: /index.php/*/media/oembed # Search Disallow: /search/ Disallow: /lubas/courses-programmes/ # Disallow all on index.php Disallow: /index.php Disallow: /index.php/ ## /en # CSS, JS, Images Allow: /en/core/*.css$ Allow: /en/core/*.css? Allow: /en/core/*.js$ Allow: /en/core/*.js? Allow: /en/core/*.gif Allow: /en/core/*.jpg Allow: /en/core/*.jpeg Allow: /en/core/*.png Allow: /en/core/*.svg Allow: /en/profiles/*.css$ Allow: /en/profiles/*.css? Allow: /en/profiles/*.js$ Allow: /en/profiles/*.js? Allow: /en/profiles/*.gif Allow: /en/profiles/*.jpg Allow: /en/profiles/*.jpeg Allow: /en/profiles/*.png Allow: /en/profiles/*.svg # Directories Disallow: /en/core/ Disallow: /en/profiles/ # Files Disallow: /en/README.md Disallow: /en/composer/Metapackage/README.txt Disallow: /en/composer/Plugin/ProjectMessage/README.md Disallow: /en/composer/Plugin/Scaffold/README.md Disallow: /en/composer/Plugin/VendorHardening/README.txt Disallow: /en/composer/Template/README.txt Disallow: /en/modules/README.txt Disallow: /en/sites/README.txt Disallow: /en/themes/README.txt Disallow: /en/web.config # Paths (clean URLs) Disallow: /en/admin/ Disallow: /en/comment/reply/ Disallow: /en/filter/tips Disallow: /en/node/add/ Disallow: /en/search/ Disallow: /en/user/register Disallow: /en/user/password Disallow: /en/user/login Disallow: /en/user/logout # Paths (no clean URLs) Disallow: /en/index.php/admin/ Disallow: /en/index.php/comment/reply/ Disallow: /en/index.php/filter/tips Disallow: /en/index.php/node/add/ Disallow: /en/index.php/search/ Disallow: /en/index.php/user/password Disallow: /en/index.php/user/register Disallow: /en/index.php/user/login Disallow: /en/index.php/user/logout # Search Disallow: /en/search/ Disallow: /en/lubas/courses-programmes/ # Disallow all on index.php Disallow: /en/index.php Disallow: /en/index.php/ ## /sv # CSS, JS, Images Allow: /sv/core/*.css$ Allow: /sv/core/*.css? Allow: /sv/core/*.js$ Allow: /sv/core/*.js? Allow: /sv/core/*.gif Allow: /sv/core/*.jpg Allow: /sv/core/*.jpeg Allow: /sv/core/*.png Allow: /sv/core/*.svg Allow: /sv/profiles/*.css$ Allow: /sv/profiles/*.css? Allow: /sv/profiles/*.js$ Allow: /sv/profiles/*.js? Allow: /sv/profiles/*.gif Allow: /sv/profiles/*.jpg Allow: /sv/profiles/*.jpeg Allow: /sv/profiles/*.png Allow: /sv/profiles/*.svg # Directories Disallow: /sv/core/ Disallow: /sv/profiles/ # Files Disallow: /sv/README.md Disallow: /sv/composer/Metapackage/README.txt Disallow: /sv/composer/Plugin/ProjectMessage/README.md Disallow: /sv/composer/Plugin/Scaffold/README.md Disallow: /sv/composer/Plugin/VendorHardening/README.txt Disallow: /sv/composer/Template/README.txt Disallow: /sv/modules/README.txt Disallow: /sv/sites/README.txt Disallow: /sv/themes/README.txt Disallow: /sv/web.config # Paths (clean URLs) Disallow: /sv/admin/ Disallow: /sv/comment/reply/ Disallow: /sv/filter/tips Disallow: /sv/node/add/ Disallow: /sv/search/ Disallow: /sv/user/register Disallow: /sv/user/password Disallow: /sv/user/login Disallow: /sv/user/logout # Paths (no clean URLs) Disallow: /sv/index.php/admin/ Disallow: /sv/index.php/comment/reply/ Disallow: /sv/index.php/filter/tips Disallow: /sv/index.php/node/add/ Disallow: /sv/index.php/search/ Disallow: /sv/index.php/user/password Disallow: /sv/index.php/user/register Disallow: /sv/index.php/user/login Disallow: /sv/index.php/user/logout # Search Disallow: /sv/search/ Disallow: /sv/lubas/courses-programmes/ # Disallow all on index.php Disallow: /sv/index.php Disallow: /sv/index.php/ ## /internal # CSS, JS, Images Allow: /internal/core/*.css$ Allow: /internal/core/*.css? Allow: /internal/core/*.js$ Allow: /internal/core/*.js? Allow: /internal/core/*.gif Allow: /internal/core/*.jpg Allow: /internal/core/*.jpeg Allow: /internal/core/*.png Allow: /internal/core/*.svg Allow: /internal/profiles/*.css$ Allow: /internal/profiles/*.css? Allow: /internal/profiles/*.js$ Allow: /internal/profiles/*.js? Allow: /internal/profiles/*.gif Allow: /internal/profiles/*.jpg Allow: /internal/profiles/*.jpeg Allow: /internal/profiles/*.png Allow: /internal/profiles/*.svg # Directories Disallow: /internal/core/ Disallow: /internal/profiles/ # Files Disallow: /internal/README.md Disallow: /internal/composer/Metapackage/README.txt Disallow: /internal/composer/Plugin/ProjectMessage/README.md Disallow: /internal/composer/Plugin/Scaffold/README.md Disallow: /internal/composer/Plugin/VendorHardening/README.txt Disallow: /internal/composer/Template/README.txt Disallow: /internal/modules/README.txt Disallow: /internal/sites/README.txt Disallow: /internal/themes/README.txt Disallow: /internal/web.config # Paths (clean URLs) Disallow: /internal/admin/ Disallow: /internal/comment/reply/ Disallow: /internal/filter/tips Disallow: /internal/node/add/ Disallow: /internal/search/ Disallow: /internal/user/register Disallow: /internal/user/password Disallow: /internal/user/login Disallow: /internal/user/logout # Paths (no clean URLs) Disallow: /internal/index.php/admin/ Disallow: /internal/index.php/comment/reply/ Disallow: /internal/index.php/filter/tips Disallow: /internal/index.php/node/add/ Disallow: /internal/index.php/search/ Disallow: /internal/index.php/user/password Disallow: /internal/index.php/user/register Disallow: /internal/index.php/user/login Disallow: /internal/index.php/user/logout # Search Disallow: /internal/search/ Disallow: /internal/lubas/courses-programmes/ # Disallow all on index.php Disallow: /internal/index.php Disallow: /internal/index.php/ ## /internt # CSS, JS, Images Allow: /internt/core/*.css$ Allow: /internt/core/*.css? Allow: /internt/core/*.js$ Allow: /internt/core/*.js? Allow: /internt/core/*.gif Allow: /internt/core/*.jpg Allow: /internt/core/*.jpeg Allow: /internt/core/*.png Allow: /internt/core/*.svg Allow: /internt/profiles/*.css$ Allow: /internt/profiles/*.css? Allow: /internt/profiles/*.js$ Allow: /internt/profiles/*.js? Allow: /internt/profiles/*.gif Allow: /internt/profiles/*.jpg Allow: /internt/profiles/*.jpeg Allow: /internt/profiles/*.png Allow: /internt/profiles/*.svg # Directories Disallow: /internt/core/ Disallow: /internt/profiles/ # Files Disallow: /internt/README.md Disallow: /internt/composer/Metapackage/README.txt Disallow: /internt/composer/Plugin/ProjectMessage/README.md Disallow: /internt/composer/Plugin/Scaffold/README.md Disallow: /internt/composer/Plugin/VendorHardening/README.txt Disallow: /internt/composer/Template/README.txt Disallow: /internt/modules/README.txt Disallow: /internt/sites/README.txt Disallow: /internt/themes/README.txt Disallow: /internt/web.config # Paths (clean URLs) Disallow: /internt/admin/ Disallow: /internt/comment/reply/ Disallow: /internt/filter/tips Disallow: /internt/node/add/ Disallow: /internt/search/ Disallow: /internt/user/register Disallow: /internt/user/password Disallow: /internt/user/login Disallow: /internt/user/logout # Paths (no clean URLs) Disallow: /internt/index.php/admin/ Disallow: /internt/index.php/comment/reply/ Disallow: /internt/index.php/filter/tips Disallow: /internt/index.php/node/add/ Disallow: /internt/index.php/search/ Disallow: /internt/index.php/user/password Disallow: /internt/index.php/user/register Disallow: /internt/index.php/user/login Disallow: /internt/index.php/user/logout # Search Disallow: /internt/search/ Disallow: /internt/lubas/courses-programmes/ # Disallow all on index.php Disallow: /internt/index.php Disallow: /internt/index.php/ ## /en/internal # CSS, JS, Images Allow: /en/internal/core/*.css$ Allow: /en/internal/core/*.css? Allow: /en/internal/core/*.js$ Allow: /en/internal/core/*.js? Allow: /en/internal/core/*.gif Allow: /en/internal/core/*.jpg Allow: /en/internal/core/*.jpeg Allow: /en/internal/core/*.png Allow: /en/internal/core/*.svg Allow: /en/internal/profiles/*.css$ Allow: /en/internal/profiles/*.css? Allow: /en/internal/profiles/*.js$ Allow: /en/internal/profiles/*.js? Allow: /en/internal/profiles/*.gif Allow: /en/internal/profiles/*.jpg Allow: /en/internal/profiles/*.jpeg Allow: /en/internal/profiles/*.png Allow: /en/internal/profiles/*.svg # Directories Disallow: /en/internal/core/ Disallow: /en/internal/profiles/ # Files Disallow: /en/internal/README.md Disallow: /en/internal/composer/Metapackage/README.txt Disallow: /en/internal/composer/Plugin/ProjectMessage/README.md Disallow: /en/internal/composer/Plugin/Scaffold/README.md Disallow: /en/internal/composer/Plugin/VendorHardening/README.txt Disallow: /en/internal/composer/Template/README.txt Disallow: /en/internal/modules/README.txt Disallow: /en/internal/sites/README.txt Disallow: /en/internal/themes/README.txt Disallow: /en/internal/web.config # Paths (clean URLs) Disallow: /en/internal/admin/ Disallow: /en/internal/comment/reply/ Disallow: /en/internal/filter/tips Disallow: /en/internal/node/add/ Disallow: /en/internal/search/ Disallow: /en/internal/user/register Disallow: /en/internal/user/password Disallow: /en/internal/user/login Disallow: /en/internal/user/logout # Paths (no clean URLs) Disallow: /en/internal/index.php/admin/ Disallow: /en/internal/index.php/comment/reply/ Disallow: /en/internal/index.php/filter/tips Disallow: /en/internal/index.php/node/add/ Disallow: /en/internal/index.php/search/ Disallow: /en/internal/index.php/user/password Disallow: /en/internal/index.php/user/register Disallow: /en/internal/index.php/user/login Disallow: /en/internal/index.php/user/logout # Search Disallow: /en/internal/search/ Disallow: /en/internal/lubas/courses-programmes/ # Disallow all on index.php Disallow: /en/internal/index.php Disallow: /en/internal/index.php/ ## /sv/internt # CSS, JS, Images Allow: /sv/internt/core/*.css$ Allow: /sv/internt/core/*.css? Allow: /sv/internt/core/*.js$ Allow: /sv/internt/core/*.js? Allow: /sv/internt/core/*.gif Allow: /sv/internt/core/*.jpg Allow: /sv/internt/core/*.jpeg Allow: /sv/internt/core/*.png Allow: /sv/internt/core/*.svg Allow: /sv/internt/profiles/*.css$ Allow: /sv/internt/profiles/*.css? Allow: /sv/internt/profiles/*.js$ Allow: /sv/internt/profiles/*.js? Allow: /sv/internt/profiles/*.gif Allow: /sv/internt/profiles/*.jpg Allow: /sv/internt/profiles/*.jpeg Allow: /sv/internt/profiles/*.png Allow: /sv/internt/profiles/*.svg # Directories Disallow: /sv/internt/core/ Disallow: /sv/internt/profiles/ # Files Disallow: /sv/internt/README.md Disallow: /sv/internt/composer/Metapackage/README.txt Disallow: /sv/internt/composer/Plugin/ProjectMessage/README.md Disallow: /sv/internt/composer/Plugin/Scaffold/README.md Disallow: /sv/internt/composer/Plugin/VendorHardening/README.txt Disallow: /sv/internt/composer/Template/README.txt Disallow: /sv/internt/modules/README.txt Disallow: /sv/internt/sites/README.txt Disallow: /sv/internt/themes/README.txt Disallow: /sv/internt/web.config # Paths (clean URLs) Disallow: /sv/internt/admin/ Disallow: /sv/internt/comment/reply/ Disallow: /sv/internt/filter/tips Disallow: /sv/internt/node/add/ Disallow: /sv/internt/search/ Disallow: /sv/internt/user/register Disallow: /sv/internt/user/password Disallow: /sv/internt/user/login Disallow: /sv/internt/user/logout # Paths (no clean URLs) Disallow: /sv/internt/index.php/admin/ Disallow: /sv/internt/index.php/comment/reply/ Disallow: /sv/internt/index.php/filter/tips Disallow: /sv/internt/index.php/node/add/ Disallow: /sv/internt/index.php/search/ Disallow: /sv/internt/index.php/user/password Disallow: /sv/internt/index.php/user/register Disallow: /sv/internt/index.php/user/login Disallow: /sv/internt/index.php/user/logout # Search Disallow: /sv/internt/search/ Disallow: /sv/internt/lubas/courses-programmes/ # Disallow all on index.php Disallow: /sv/internt/index.php Disallow: /sv/internt/index.php/