Charger vos sites dans le cache Varnish avec le sitemap (crawler)

#!/bin/bash
 
Sitemap='www.exemple.com/sitemap.xml'
Site='www.exemple.com'
wget='/usr/bin/wget'
grep='/bin/grep'
awk='/usr/bin/awk'
find='/usr/bin/find'
mount='/bin/mount'
umount='/bin/umount'
sed='/bin/sed'
wc='/usr/bin/wc'
cut='/usr/bin/cut'
 
if [ `$mount -l | $grep /tmp/wget | $wc -l` -gt 0 ] ; then exit 1 ; fi
 
touch /tmp/cookies.txt
$wget -q -O /dev/null "URL pour régénérer le sitemap"
 
if [ ! -d /tmp/wget ] ; then
   mkdir /tmp/wget
fi
 
$mount -t tmpfs -o size=25M tmpfs /tmp/wget
cd /tmp/wget
 
for site in $Sitemap ; do
   for url in `$wget -q -O - "http://$site" | $grep '<loc>'  | $awk -F'<loc>' '{print $2}' | $awk -F'</loc>' '{print $1}'` ; do
      $wget -q -p --save-cookies /tmp/cookies.txt --load-cookies /tmp/cookies.txt "$url"
      for index in `$find . -type f | $sed 's:^./::' | $grep index.html` ; do
            for href in `$grep href "$index" | $cut -d"'" -f2 | $cut -d'"' -f2 | $grep http | $grep "$Site"` ; do
                  $wget -q -p --save-cookies /tmp/cookies.txt --load-cookies /tmp/cookies.txt --header='Accept-Encoding: gzip, deflate' "$href"
                  $wget -q -p --header='Accept-Encoding: gzip, deflate' "$href"
            done
      done
      for file in `$find . -type f | $sed 's:^./::' | $grep -v index.html` ; do
          $wget -q -O /dev/null  --save-cookies /tmp/cookies.txt --load-cookies /tmp/cookies.txt --header='Accept-Encoding: gzip, deflate' "http://$file"
          $wget -q -O /dev/null  --header='Accept-Encoding: gzip, deflate' "http://$file"
      done
      for file in `$find . -type f | $sed 's:^./::' | $grep index.html | $sed 's:index.html::'` ; do
          $wget -q -O /dev/null  --save-cookies /tmp/cookies.txt --load-cookies /tmp/cookies.txt --header='Accept-Encoding: gzip, deflate' "http://$file"
          $wget -q -O /dev/null  --header='Accept-Encoding: gzip, deflate' "http://$file"
      done
      rm -rf /tmp/wget/*
   done
done
rm /tmp/cookies.txt
cd /tmp/ ; $umount /tmp/wget ; rm -rf /tmp/wget