using System;
using System.IO;
using System.Net;

class WebCrawler
{
    public static void Main(String[] args)
    {
        String text = fileToText(args[0]);
        text = text.Replace('\r', ' ');
        String[] urls = text.Split('\n');
        for (int i = 0; i < urls.Length; i++)
        {
            String url = urls[i].Trim();
            String fileName = toFileName(url);
            Console.WriteLine(i + ":url=" + urls[i]+" file="+fileName);
            urlToFile(urls[i], "data/"+fileName);
        }
    }

    public static String fileToText(String filePath)
    {
        StreamReader file = new StreamReader(filePath);
        String text = file.ReadToEnd();
        file.Close();
        return text;
    }

    public static void urlToFile(String url, String file) {
        WebClient webclient = new WebClient();
        webclient.DownloadFile("http://"+url, file);
    }

    public static String toFileName(String url)
    {
        String fileName = url.Replace('?', '_');
        fileName = fileName.Replace('/', '_');
        fileName = fileName.Replace('&', '_');
        fileName = fileName.ToLower();
        if (!fileName.EndsWith(".htm") && !fileName.EndsWith(".html"))
            fileName = fileName + ".htm";
        return fileName;
    }
}