refactor: added `robots.txt` to disallow indexing

gulpfile.js

@@ -48,6 +48,10 @@ let paths = {
     {
       src: 'src_data/images/*.*',
       dest: 'data/static/images/'
+    },
+    {
+      src: 'src_data/*.txt',
+      dest: 'data/static/'
     }
   ],
   pages: {

src/PortalTask.h

@@ -859,6 +859,7 @@ protected:
       }
     });
 
+    this->webServer->serveStatic("/robots.txt", LittleFS, "/static/robots.txt", PORTAL_CACHE);
     this->webServer->serveStatic("/favicon.ico", LittleFS, "/static/images/favicon.ico", PORTAL_CACHE);
     this->webServer->serveStatic("/static", LittleFS, "/static", PORTAL_CACHE);
   }

src_data/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /