unit Main;

interface

uses
  Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
  Dialogs, Menus, About, ImgList, ComCtrls, ToolWin, StdCtrls, ExtCtrls,
  SkinCaption, WinSkinData, AddNewSite, ShellAPI, SCImageBox,
  SCStdControls, SCPanels, SCControl, BigINI, ChangeSettings, Processing,
  IdFTP, IdBaseComponent, IdComponent, IdTCPConnection, IdTCPClient, IdHTTP,
  Report, RegExpr, StrUtils;

const MAXSITES = 1024;

type
  TFTPSettings = record
    Hostname,
    Port,
    UserName,
    Password,
    Directory: String;
    PassiveTransfer,
    UseProxy: Boolean;
    ProxyHostname,
    ProxyPort,
    ProxyUsername,
    ProxyPassword: String;
    ProxyType: Integer;
  end;
  TWebSettings = record
    WebsiteName,
    MainKeyword,
    AdsensePub,
    ChitikaPub,
    ChitikaChannel,
    PeakClickAffID,
    PeackClickSubaffID,
    NrOfSearchResults,
    CreatedAt,
//    PublicHTML,
    URL: String;
    DisableThumbs: Boolean;
  end;
  TKeywordSettings = record
    AutoScrapeKeyword,
    UseOwnKeywords: Boolean;
    StartingKeyword,
    MaxNrOfKeywords,
    KeywordFile: String;
    UseOwnArticles,
    AutoGenerateArticles: Boolean;
    ArticleFolder,
    ArticlesPerKeyword,
    WordsPerArticle,
    MarkovGranularity: String;
  end;
  TSiteDetails = record
    FTP: TFTPSettings;
    Settings: TWebSettings;
    Keyword: TKeywordSettings;
  end;

  TMainForm = class(TForm)
    MainMenu: TMainMenu;
    File1: TMenuItem;
    NewProject1: TMenuItem;
    SaveProject1: TMenuItem;
    SaveProjectAs1: TMenuItem;
    OpenProject1: TMenuItem;
    N1: TMenuItem;
    N2: TMenuItem;
    Exit1: TMenuItem;
    Project1: TMenuItem;
    Help1: TMenuItem;
    Contents1: TMenuItem;
    N3: TMenuItem;
    About1: TMenuItem;
    ToolBar1: TToolBar;
    btnNewSiteWizard: TToolButton;
    ToolbarHotImages: TImageList;
    btnNewProject: TToolButton;
    btnSaveProject: TToolButton;
    btnSaveProjectAs: TToolButton;
    btnChangeSiteSettings: TToolButton;
    btnProcessAll: TToolButton;
    btnAbout: TToolButton;
    btnHelp: TToolButton;
    Panel1: TPanel;
    Panel3: TPanel;
    SitesList: TListBox;
    Label1: TLabel;
    Bevel1: TBevel;
    ToolbarSimpleImages: TImageList;
    SkinData: TSkinData;
    SkinCaption: TSkinCaption;
    AddNewSite1: TMenuItem;
    Splitter1: TSplitter;
    N5: TMenuItem;
    ChangeSiteSettings1: TMenuItem;
    ToolButton1: TToolButton;
    btnOpenProject: TToolButton;
    ToolButton2: TToolButton;
    ToolButton3: TToolButton;
    ToolButton4: TToolButton;
    SCGroupContainer1: TSCGroupContainer;
    ControlPanel: TSCAdvPanel;
    SCImageBox4: TSCImageBox;
    SCImageBox5: TSCImageBox;
    SCImageBox6: TSCImageBox;
    SCImageBox8: TSCImageBox;
    SCLabel2: TSCLabel;
    SCLabel3: TSCLabel;
    SCLabel4: TSCLabel;
    SCLabel5: TSCLabel;
    SCAdvPanel3: TSCAdvPanel;
    lblKnownThreats: TSCLabel;
    SCImageBox2: TSCImageBox;
    lblName: TSCLabel;
    btnRemoveSite: TToolButton;
    RemoveSelectedSite1: TMenuItem;
    SaveDialog: TSaveDialog;
    OpenDialog: TOpenDialog;
    SCLabel6: TSCLabel;
    lblURL: TSCLabel;
    SCLabel8: TSCLabel;
    lblMainKeyword: TSCLabel;
    SCLabel1: TSCLabel;
    lblCreatedAt: TSCLabel;
    SCLabel7: TSCLabel;
    lblAdsensePub: TSCLabel;
    btnMoveUp: TButton;
    btnMoveDown: TButton;
    lblStatus: TLabel;
    N4: TMenuItem;
    ProcessAllWebsites1: TMenuItem;
    HTTP: TIdHTTP;
    FTP: TIdFTP;
    procedure Exit1Click(Sender: TObject);
    procedure About1Click(Sender: TObject);
    procedure AddNewSite1Click(Sender: TObject);
    procedure Contents1Click(Sender: TObject);
    procedure NewProject1Click(Sender: TObject);
    procedure SaveProject1Click(Sender: TObject);
    procedure OpenProject1Click(Sender: TObject);
    procedure SaveProjectAs1Click(Sender: TObject);
    procedure SitesListDrawItem(Control: TWinControl; Index: Integer;
      Rect: TRect; State: TOwnerDrawState);
    procedure FormCreate(Sender: TObject);
    procedure SitesListClick(Sender: TObject);
    procedure ChangeSiteSettings1Click(Sender: TObject);
    procedure RemoveSelectedSite1Click(Sender: TObject);
    procedure btnMoveUpClick(Sender: TObject);
    procedure btnMoveDownClick(Sender: TObject);
    procedure FormCloseQuery(Sender: TObject; var CanClose: Boolean);
    procedure ProcessAllWebsites1Click(Sender: TObject);
  private
    { Private declarations }
    function ExtractFileName(FileName: String): String;
    function ExtractFilePath(FileName: String): String;

    // used for processing
    procedure CopyRSSGMDirToTempDir;
    procedure ApplySiteConfiguration(Index: Integer);
    procedure ConfigureKeywords(Index: Integer);
    procedure ConfigureArticles(Index: Integer);
    procedure UploadSite(Index: Integer);
    procedure AssignFTPProperties(Index: Integer);
    procedure AutoGenerateArticles(Index: Integer);
    function BuildSite(Index: Integer): String;
    procedure AutoScrapeKeywords(Index: Integer; SaveToFilename: String);
  public
    { Public declarations }

    // current-project
    CurrentProjectName: String;
    Untitled, Modified: Boolean;

    // project-related
    Sites: array[0..MAXSITES-1] of TSiteDetails;
    NrOfSites: Integer;

    // methods
    procedure NewProject;
    procedure OpenProject(FileName: String);
    procedure SaveProject(FileName: String);
  end;

var
  MainForm: TMainForm;

implementation

uses Types;

{$R *.dfm}

procedure TMainForm.Exit1Click(Sender: TObject);
begin
  Close;
end;

procedure TMainForm.About1Click(Sender: TObject);
begin
  with TAboutForm.Create(Self) do
  try
   ShowModal;
  finally
   Free;
  end;
end;

function RemoveQuotes(S: String): String;
begin
  if S<>'' then
   if S[1]='"' then
    Delete(S, 1, 1);
  if S<>'' then
   if S[Length(S)]='"' then
    Delete(S, Length(S), 1);
  Result := S;
end;

procedure TMainForm.AddNewSite1Click(Sender: TObject);
var I: Integer;
begin
  with TAddNewSiteWizardForm.Create(Self) do
  try
   ShowModal;

   if not Cancelled then
   begin

   Modified := True;
   Caption := ExtractFileName(CurrentProjectName)+'* - Web Site Sizzler';

   Inc(NrOfSites);
   I := NrOfSites-1;

   Sites[I].Settings.WebsiteName := WebsiteName.Text;
   Sites[I].Settings.MainKeyword := MainKeyword.Text;
   Sites[I].Settings.AdsensePub := AdsensePub.Text;
   Sites[I].Settings.ChitikaPub := ChitikaPub.Text;
   Sites[I].Settings.ChitikaChannel := ChitikaChannel.Text;
   Sites[I].Settings.PeakClickAffID := AffID.Text;
   Sites[I].Settings.PeackClickSubaffID := SubaffID.Text;
   Sites[I].Settings.NrOfSearchResults := NoSearchResults.Text;
   Sites[I].Settings.DisableThumbs := chkDisableThumbs.Checked;
//   Sites[I].Settings.PublicHTML := PublicHTML.Text;
   Sites[I].Settings.URL := WebsiteURL.Text;
   Sites[I].Settings.CreatedAt := DateTimeToStr(Now);

   Sites[I].Keyword.MaxNrOfKeywords := IntToStr(MaxKeywords.Value);
   Sites[I].Keyword.AutoScrapeKeyword := chkAutoScrape.Checked;
   Sites[I].Keyword.UseOwnKeywords := chkMyScrape.Checked;
   Sites[I].Keyword.StartingKeyword := MyKeyword.Text;
   Sites[I].Keyword.KeywordFile := RemoveQuotes(KeywordsFile.Text);
   Sites[I].Keyword.UseOwnArticles := chkUseOwnArticles.Checked;
   Sites[I].Keyword.AutoGenerateArticles := chkAutoGenerateArticles.Checked;
   Sites[I].Keyword.ArticleFolder := RemoveQuotes(ArticlesFolder.Text);
   Sites[I].Keyword.ArticlesPerKeyword := IntToStr(ArticlesPerKeyword.Value);
   Sites[I].Keyword.WordsPerArticle := IntToStr(WordsPerArticle.Value);
   Sites[I].Keyword.MarkovGranularity := MarkovGranularity.Text;

   Sites[I].FTP.Hostname := FTPHostname.Text;
   Sites[I].FTP.Port := FTPPort.Text;
   Sites[I].FTP.UserName := FTPUserName.Text;
   Sites[I].FTP.Password := FTPPassword.Text;
   Sites[I].FTP.Directory := FTPDirectory.Text;
   Sites[I].FTP.PassiveTransfer := FTPPassive.Checked;
   Sites[I].FTP.UseProxy := FTPUseProxy.Checked;
   Sites[I].FTP.ProxyHostname := FTPProxyHostname.Text;
   Sites[I].FTP.ProxyPort := FTPProxyPort.Text;
   Sites[I].FTP.ProxyUsername := FTPProxyUsername.Text;
   Sites[I].FTP.ProxyPassword := FTPProxyPassword.Text;
   Sites[I].FTP.ProxyType := FTPProxyType.ItemIndex;

   // add this site to the list
   SitesList.Items.Add(Sites[I].Settings.WebsiteName+'~'+Sites[I].Settings.CreatedAt);

   end;
  finally
   Free;
  end;
end;

procedure TMainForm.Contents1Click(Sender: TObject);
begin
  ShellExecute(Handle, 'open', 'http://www.websitesizzler.com/help', '', '', SW_SHOWNORMAL);
end;

procedure TMainForm.NewProject1Click(Sender: TObject);
begin
  if Modified then
   if Application.MessageBox('Would you like to save your work before creating a new project?', 'Confirm', MB_YESNO+MB_ICONQUESTION) = mrYes then
    SaveProject1Click(Self);
  NewProject;
end;

procedure TMainForm.SaveProject1Click(Sender: TObject);
begin
  if Untitled then
   SaveProjectAs1Click(Self)
  else
   SaveProject(CurrentProjectName);
end;

procedure TMainForm.NewProject;
var I: Integer;
begin
  Caption := 'Untitled - Web Site Sizzler';
  Untitled := True;
  Modified := False;
  CurrentProjectName := 'Untitled';
  lblStatus.Caption := 'There are no websites in the current project.';

  NrOfSites := 0;
  for I:=0 to MAXSITES-1 do
   FillChar(Sites[I], SizeOf(TSiteDetails), #0);

  SitesList.Items.Clear;
end;

procedure TMainForm.OpenProject1Click(Sender: TObject);
begin
  if Modified then
   if Application.MessageBox('Would you like to save your work before opening a project?', 'Confirm', MB_YESNO+MB_ICONQUESTION) = mrYes then
    SaveProject1Click(Self);
  if OpenDialog.Execute then
   OpenProject(OpenDialog.FileName);
end;

procedure TMainForm.OpenProject(FileName: String);
var INI: TBigINIFile;
    I: Integer;
begin
  INI := TBigINIFile.Create(FileName);

  Untitled := False;
  CurrentProjectName := FileName;
  Modified := False;
  Caption := ExtractFileName(CurrentProjectName)+' - Web Site Sizzler';

  NrOfSites := INI.ReadInteger('General', 'NrOfSites', 0);
  if NrOfSites = 1 then
   lblStatus.Caption := 'There is one website in the current project.'
  else
   lblStatus.Caption := 'There are '+FloatToStrF(NrOfSites, ffNumber, 18, 0)+' websites in the current project.';
  SitesList.Items.BeginUpdate;
  SitesList.Items.Clear;
  for I:=0 to NrOfSites-1 do
  begin
    Sites[I].Settings.WebsiteName := INI.ReadString('Site'+IntToStr(I), 'WebsiteName', '<no name specified>');
    Sites[I].Settings.MainKeyword := INI.ReadString('Site'+IntToStr(I), 'MainKeyword', '');
    Sites[I].Settings.AdsensePub := INI.ReadString('Site'+IntToStr(I), 'AdsensePub', '');
    Sites[I].Settings.ChitikaPub := INI.ReadString('Site'+IntToStr(I), 'ChitikaPub', '');
    Sites[I].Settings.ChitikaChannel := INI.ReadString('Site'+IntToStr(I), 'ChitikaChannel', '');
    Sites[I].Settings.PeakClickAffID := INI.ReadString('Site'+IntToStr(I), 'PeakClickAffID', '');
    Sites[I].Settings.PeackClickSubaffID := INI.ReadString('Site'+IntToStr(I), 'PeakClickSubaffID', '');
    Sites[I].Settings.NrOfSearchResults := INI.ReadString('Site'+IntToStr(I), 'NrOfSearchResults', '');
//    Sites[I].Settings.PublicHTML := INI.ReadString('Site'+IntToStr(I), 'PublicHTML', '');
    Sites[I].Settings.URL := INI.ReadString('Site'+IntToStr(I), 'URL', '');
    Sites[I].Settings.DisableThumbs := INI.ReadBool('Site'+IntToStr(I), 'DisableThumbs', False);
    Sites[I].Settings.CreatedAt := INI.ReadString('Site'+IntToStr(I), 'CreatedAt', DateTimeToStr(Now));

    Sites[I].Keyword.MaxNrOfKeywords := INI.ReadString('Site'+IntToStr(I), 'MaxNrOfKeywords', '1000');
    Sites[I].Keyword.AutoScrapeKeyword := INI.ReadBool('Site'+IntToStr(I), 'AutoscrapeKeyword', True);
    Sites[I].Keyword.UseOwnKeywords := INI.ReadBool('Site'+IntToStr(I), 'UseOwnKeywords', False);
    Sites[I].Keyword.StartingKeyword := INI.ReadString('Site'+IntToStr(I), 'StartingKeyword', '');
    Sites[I].Keyword.KeywordFile := INI.ReadString('Site'+IntToStr(I), 'KeywordFile', '');
    Sites[I].Keyword.UseOwnArticles := INI.ReadBool('Site'+IntToStr(I), 'UseOwnArticles', True);
    Sites[I].Keyword.AutoGenerateArticles := INI.ReadBool('Site'+IntToStr(I), 'AutoGenerateArticles', False);
    Sites[I].Keyword.ArticleFolder := INI.ReadString('Site'+IntToStr(I), 'ArticleFolder', '');
    Sites[I].Keyword.ArticlesPerKeyword := INI.ReadString('Site'+IntToStr(I), 'ArticlesPerKeyword', '');
    Sites[I].Keyword.WordsPerArticle := INI.ReadString('Site'+IntToStr(I), 'WordsPerArticle', '');
    Sites[I].Keyword.MarkovGranularity := INI.ReadString('Site'+IntToStr(I), 'MarkovGranularity', '');

    Sites[I].FTP.Hostname := INI.ReadString('Site'+IntToStr(I), 'Hostname', '');
    Sites[I].FTP.Port := INI.ReadString('Site'+IntToStr(I), 'Port', '21');
    Sites[I].FTP.UserName := INI.ReadString('Site'+IntToStr(I), 'UserName', '');
    Sites[I].FTP.Password := INI.ReadString('Site'+IntToStr(I), 'Password', '');
    Sites[I].FTP.Directory := INI.ReadString('Site'+IntToStr(I), 'Directory', '');
    Sites[I].FTP.PassiveTransfer := INI.ReadBool('Site'+IntToStr(I), 'PassiveTransfer', False);
    Sites[I].FTP.UseProxy := INI.ReadBool('Site'+IntToStr(I), 'UseProxy', False);
    Sites[I].FTP.ProxyHostname := INI.ReadString('Site'+IntToStr(I), 'ProxyHostname', '');
    Sites[I].FTP.ProxyPort := INI.ReadString('Site'+IntToStr(I), 'ProxyPort', '');
    Sites[I].FTP.ProxyUsername := INI.ReadString('Site'+IntToStr(I), 'ProxyUsername', '');
    Sites[I].FTP.ProxyPassword := INI.ReadString('Site'+IntToStr(I), 'ProxyPassword', '');
    Sites[I].FTP.ProxyType := INI.ReadInteger('Site'+IntToStr(I), 'ProxyType', 0);

    // add this site to the list
    SitesList.Items.Add(Sites[I].Settings.WebsiteName+'~'+Sites[I].Settings.CreatedAt);
  end;
  SitesList.Items.EndUpdate;
  INI.Free;
end;

procedure TMainForm.SaveProjectAs1Click(Sender: TObject);
begin
  if SaveDialog.Execute then
   SaveProject(SaveDialog.FileName);
end;

procedure TMainForm.SaveProject(FileName: String);
var INI: TBigINIFile;
    I: Integer;
begin
  // delete old file
  DeleteFile(FileName);

  INI := TBigINIFile.Create(FileName);

  Caption := ExtractFileName(FileName)+' - Web Site Sizzler';
  Untitled := False;
  CurrentProjectName := FileName;
  Modified := False;
  Caption := ExtractFileName(CurrentProjectName)+' - Web Site Sizzler';

  INI.WriteInteger('General', 'NrOfSites', NrOfSites);
  for I:=0 to NrOfSites-1 do
  begin
    INI.WriteString('Site'+IntToStr(I), 'WebsiteName', Sites[I].Settings.WebsiteName);
    INI.WriteString('Site'+IntToStr(I), 'MainKeyword', Sites[I].Settings.MainKeyword);
    INI.WriteString('Site'+IntToStr(I), 'AdsensePub', Sites[I].Settings.AdsensePub);
    INI.WriteString('Site'+IntToStr(I), 'ChitikaPub', Sites[I].Settings.ChitikaPub);
    INI.WriteString('Site'+IntToStr(I), 'ChitikaChannel', Sites[I].Settings.ChitikaChannel);
    INI.WriteString('Site'+IntToStr(I), 'PeakClickAffID', Sites[I].Settings.PeakClickAffID);
    INI.WriteString('Site'+IntToStr(I), 'PeakClickSubaffID', Sites[I].Settings.PeackClickSubaffID);
    INI.WriteString('Site'+IntToStr(I), 'NrOfSearchResults', Sites[I].Settings.NrOfSearchResults);
//    INI.WriteString('Site'+IntToStr(I), 'PublicHTML', Sites[I].Settings.PublicHTML);
    INI.WriteString('Site'+IntToStr(I), 'URL', Sites[I].Settings.URL);
    INI.WriteBool('Site'+IntToStr(I), 'DisableThumbs', Sites[I].Settings.DisableThumbs);
    INI.WriteString('Site'+IntToStr(I), 'CreatedAt', Sites[I].Settings.CreatedAt);

    INI.WriteString('Site'+IntToStr(I), 'MaxNrOfKeywords', Sites[I].Keyword.MaxNrOfKeywords);
    INI.WriteBool('Site'+IntToStr(I), 'AutoscrapeKeyword', Sites[I].Keyword.AutoScrapeKeyword);
    INI.WriteBool('Site'+IntToStr(I), 'UseOwnKeywords', Sites[I].Keyword.UseOwnKeywords);
    INI.WriteString('Site'+IntToStr(I), 'StartingKeyword', Sites[I].Keyword.StartingKeyword);
    INI.WriteString('Site'+IntToStr(I), 'KeywordFile', Sites[I].Keyword.KeywordFile);
    INI.WriteBool('Site'+IntToStr(I), 'UseOwnArticles', Sites[I].Keyword.UseOwnArticles);
    INI.WriteBool('Site'+IntToStr(I), 'AutoGenerateArticles', Sites[I].Keyword.AutoGenerateArticles);
    INI.WriteString('Site'+IntToStr(I), 'ArticleFolder', Sites[I].Keyword.ArticleFolder);
    INI.WriteString('Site'+IntToStr(I), 'ArticlesPerKeyword', Sites[I].Keyword.ArticlesPerKeyword);
    INI.WriteString('Site'+IntToStr(I), 'WordsPerArticle', Sites[I].Keyword.WordsPerArticle);
    INI.WriteString('Site'+IntToStr(I), 'MarkovGranularity', Sites[I].Keyword.MarkovGranularity);

    INI.WriteString('Site'+IntToStr(I), 'Hostname', Sites[I].FTP.Hostname);
    INI.WriteString('Site'+IntToStr(I), 'Port', Sites[I].FTP.Port);
    INI.WriteString('Site'+IntToStr(I), 'UserName', Sites[I].FTP.UserName);
    INI.WriteString('Site'+IntToStr(I), 'Password', Sites[I].FTP.Password);
    INI.WriteString('Site'+IntToStr(I), 'Directory', Sites[I].FTP.Directory);
    INI.WriteBool('Site'+IntToStr(I), 'PassiveTransfer', Sites[I].FTP.PassiveTransfer);
    INI.WriteBool('Site'+IntToStr(I), 'UseProxy', Sites[I].FTP.UseProxy);
    INI.WriteString('Site'+IntToStr(I), 'ProxyHostname', Sites[I].FTP.ProxyHostname);
    INI.WriteString('Site'+IntToStr(I), 'ProxyPort', Sites[I].FTP.ProxyPort);
    INI.WriteString('Site'+IntToStr(I), 'ProxyUsername', Sites[I].FTP.ProxyUsername);
    INI.WriteString('Site'+IntToStr(I), 'ProxyPassword', Sites[I].FTP.ProxyPassword);
    INI.WriteInteger('Site'+IntToStr(I), 'ProxyType', Sites[I].FTP.ProxyType);
  end;
  INI.Free;
end;

procedure TMainForm.SitesListDrawItem(Control: TWinControl; Index: Integer;
  Rect: TRect; State: TOwnerDrawState);
var
  bitmap : TBitmap;
  alist: TListBox;
  S, S1: String;
begin
 try
  { type-cast the Control object }
  alist := Control as TListBox;
  { get the bitmap from the TStringList object storage }
  bitmap := TBitmap.Create;
  if Index mod 2 = 0 then
   ToolbarHotImages.GetBitmap(11, bitmap)
  else
   ToolbarHotImages.GetBitmap(18, bitmap);
  { erase the current rectangle -- this MUST be done or you get garbage}
  { from previous drawing, trust me...}
  if Index mod 2 = 0 then
   alist.Canvas.Brush.Color := clWhite
  else
   alist.Canvas.Brush.Color := $00EAEAFF;
  alist.Canvas.FillRect(Rect);

  { draw the bitmap }
  alist.Canvas.Draw(Rect.Left, Rect.Top, bitmap);

  { draw the text }
  S := alist.Items[Index];
  S1 := Copy(S, 1, Pos('~',S)-1);
  Delete(S, 1, Length(S1)+1);
  alist.Canvas.Font.Style := [fsBold];
  alist.Canvas.Font.Size := 12;
  alist.Canvas.Font.Color := clNavy;
  alist.Canvas.TextOut(Rect.Left + bitmap.Width + 4, Rect.Top + 8, S1);
  alist.Canvas.Font.Style := [];
  alist.Canvas.Font.Size := 9;
  alist.Canvas.Font.Color := clMaroon;
  alist.Canvas.TextOut(Rect.Left + bitmap.Width + 4, Rect.Top + 25, S);
 except
 end;
end;

procedure TMainForm.FormCreate(Sender: TObject);
begin
  NewProject;
end;

procedure TMainForm.SitesListClick(Sender: TObject);
var Index: Integer;
begin
  Index := SitesList.ItemIndex;
  if Index < 0 then
  begin
    lblName.Caption := 'N/A';
    lblURL.Caption := 'N/A';
    lblCreatedAt.Caption := 'N/A';
    lblMainKeyword.Caption := 'N/A';
    lblAdsensePub.Caption := 'N/A';
    Exit;
  end;
  lblName.Caption := Sites[Index].Settings.WebsiteName;
  lblURL.Caption := Sites[Index].Settings.URL;
  lblCreatedAt.Caption := Sites[Index].Settings.CreatedAt;
  lblMainKeyword.Caption := Sites[Index].Settings.MainKeyword;
  lblAdsensePub.Caption := Sites[Index].Settings.AdsensePub;
end;

procedure TMainForm.ChangeSiteSettings1Click(Sender: TObject);
var I: Integer;
begin
  I := SitesList.ItemIndex;
  if I >= 0 then
  with TChangeSiteSettingsForm.Create(Self) do
  try
   WebsiteName.Text := Sites[I].Settings.WebsiteName;
   MainKeyword.Text := Sites[I].Settings.MainKeyword;
   AdsensePub.Text := Sites[I].Settings.AdsensePub;
   ChitikaPub.Text := Sites[I].Settings.ChitikaPub;
   ChitikaChannel.Text := Sites[I].Settings.ChitikaChannel;
   AffID.Text := Sites[I].Settings.PeakClickAffID;
   SubaffID.Text := Sites[I].Settings.PeackClickSubaffID;
   NoSearchResults.Text := Sites[I].Settings.NrOfSearchResults;
   WebsiteURL.Text := Sites[I].Settings.URL;
//   PublicHTML.Text := Sites[I].Settings.PublicHTML;
   chkDisableThumbs.Checked := Sites[I].Settings.DisableThumbs;

   MaxKeywords.Value := StrToInt(Sites[I].Keyword.MaxNrOfKeywords);
   chkAutoScrape.Checked := Sites[I].Keyword.AutoScrapeKeyword;
   chkMyScrape.Checked := Sites[I].Keyword.UseOwnKeywords;
   MyKeyword.Text := Sites[I].Keyword.StartingKeyword;
   KeywordsFile.Text := Sites[I].Keyword.KeywordFile;
   chkUseOwnArticles.Checked := Sites[I].Keyword.UseOwnArticles;
   chkAutoGenerateArticles.Checked := Sites[I].Keyword.AutoGenerateArticles;
   ArticlesFolder.Text := Sites[I].Keyword.ArticleFolder;
   try
    ArticlesPerKeyword.Value := StrToInt(Sites[I].Keyword.ArticlesPerKeyword);
   except
    ArticlesPerKeyword.Value := 0;
   end;
   try
    WordsPerArticle.Value := StrToInt(Sites[I].Keyword.WordsPerArticle);
   except
    WordsPerArticle.Value := 0;
   end;
   MarkovGranularity.Text := Sites[I].Keyword.MarkovGranularity;

   FTPHostname.Text := Sites[I].FTP.Hostname;
   FTPPort.Text := Sites[I].FTP.Port;
   FTPUserName.Text := Sites[I].FTP.UserName;
   FTPPassword.Text := Sites[I].FTP.Password;
   FTPDirectory.Text := Sites[I].FTP.Directory;
   FTPPassive.Checked := Sites[I].FTP.PassiveTransfer;
   FTPUseProxy.Checked := Sites[I].FTP.UseProxy;
   FTPProxyHostname.Text := Sites[I].FTP.ProxyHostname;
   FTPProxyPort.Text := Sites[I].FTP.ProxyPort;
   FTPProxyUsername.Text := Sites[I].FTP.ProxyUsername;
   FTPProxyPassword.Text := Sites[I].FTP.ProxyPassword;
   FTPProxyType.ItemIndex := Sites[I].FTP.ProxyType;

   ShowModal;
   if not Cancelled then
   begin

   Modified := True;
   Caption := ExtractFileName(CurrentProjectName)+'* - Web Site Sizzler';

   Sites[I].Settings.WebsiteName := WebsiteName.Text;
   Sites[I].Settings.MainKeyword := MainKeyword.Text;
   Sites[I].Settings.AdsensePub := AdsensePub.Text;
   Sites[I].Settings.ChitikaPub := ChitikaPub.Text;
   Sites[I].Settings.ChitikaChannel := ChitikaChannel.Text;
   Sites[I].Settings.PeakClickAffID := AffID.Text;
   Sites[I].Settings.PeackClickSubaffID := SubaffID.Text;
   Sites[I].Settings.NrOfSearchResults := NoSearchResults.Text;
   Sites[I].Settings.URL := WebsiteURL.Text;
//   Sites[I].Settings.PublicHTML := PublicHTML.Text;
   Sites[I].Settings.DisableThumbs := chkDisableThumbs.Checked;
   Sites[I].Settings.CreatedAt := DateTimeToStr(Now);

   Sites[I].Keyword.MaxNrOfKeywords := IntToStr(MaxKeywords.Value);
   Sites[I].Keyword.AutoScrapeKeyword := chkAutoScrape.Checked;
   Sites[I].Keyword.UseOwnKeywords := chkMyScrape.Checked;
   Sites[I].Keyword.StartingKeyword := MyKeyword.Text;
   Sites[I].Keyword.KeywordFile := RemoveQuotes(KeywordsFile.Text);
   Sites[I].Keyword.UseOwnArticles := chkUseOwnArticles.Checked;
   Sites[I].Keyword.AutoGenerateArticles := chkAutoGenerateArticles.Checked;
   Sites[I].Keyword.ArticleFolder := RemoveQuotes(ArticlesFolder.Text);
   Sites[I].Keyword.ArticlesPerKeyword := IntToStr(ArticlesPerKeyword.Value);
   Sites[I].Keyword.WordsPerArticle := IntToStr(WordsPerArticle.Value);
   Sites[I].Keyword.MarkovGranularity := MarkovGranularity.Text;

   Sites[I].FTP.Hostname := FTPHostname.Text;
   Sites[I].FTP.Port := FTPPort.Text;
   Sites[I].FTP.UserName := FTPUserName.Text;
   Sites[I].FTP.Password := FTPPassword.Text;
   Sites[I].FTP.Directory := FTPDirectory.Text;
   Sites[I].FTP.PassiveTransfer := FTPPassive.Checked;
   Sites[I].FTP.UseProxy := FTPUseProxy.Checked;
   Sites[I].FTP.ProxyHostname := FTPProxyHostname.Text;
   Sites[I].FTP.ProxyPort := FTPProxyPort.Text;
   Sites[I].FTP.ProxyUsername := FTPProxyUsername.Text;
   Sites[I].FTP.ProxyPassword := FTPProxyPassword.Text;
   Sites[I].FTP.ProxyType := FTPProxyType.ItemIndex;

   // add this site to the list
   SitesList.Items[I] := Sites[I].Settings.WebsiteName+'~'+Sites[I].Settings.CreatedAt;
   end;
  finally
   Free;
  end;

  // simulate on-click
  SitesListClick(Self);
end;

procedure TMainForm.RemoveSelectedSite1Click(Sender: TObject);
var I, J: Integer;
begin
  if SitesList.ItemIndex < 0 then Exit;
  if Application.MessageBox(PChar('Are you sure you want to delete the site called "'+lblName.Caption+'"?'), 'Confirm Delete', MB_YESNO+MB_ICONQUESTION) = mrYes then
  begin
    Modified := True;
    Caption := ExtractFileName(CurrentProjectName)+'* - Web Site Sizzler';

    // delete from list
    J := SitesList.ItemIndex;
    for I := J to NrOfSites-2 do
      Sites[I] := Sites[I+1];
    Dec(NrOfSites);

    // delete from list
    SitesList.Items.Delete(J);

    // simulate on-click
    SitesListClick(Self);

    if NrOfSites = 0 then
     lblStatus.Caption := 'There are no websites in the current project.'
    else
    if NrOfSites = 1 then
     lblStatus.Caption := 'There is one website in the current project.'
    else
     lblStatus.Caption := 'There are '+FloatToStrF(NrOfSites, ffNumber, 18, 0)+' websites in the current project.';
  end;
end;

procedure TMainForm.btnMoveUpClick(Sender: TObject);
var Aux: TSiteDetails;
begin
  if SitesList.ItemIndex <= 0 then Exit;
  Aux := Sites[SitesList.ItemIndex];
  Sites[SitesList.ItemIndex] := Sites[SitesList.ItemIndex-1];
  Sites[SitesList.ItemIndex-1] := Aux;
  SitesList.Items.Exchange(SitesList.ItemIndex, SitesList.ItemIndex-1);
  Modified := True;
  Caption := ExtractFileName(CurrentProjectName)+'* - Web Site Sizzler';
end;

procedure TMainForm.btnMoveDownClick(Sender: TObject);
var Aux: TSiteDetails;
begin
  if SitesList.ItemIndex < 0 then Exit;
  if SitesList.ItemIndex = SitesList.Items.Count-1 then Exit;
  Aux := Sites[SitesList.ItemIndex];
  Sites[SitesList.ItemIndex] := Sites[SitesList.ItemIndex+1];
  Sites[SitesList.ItemIndex+1] := Aux;
  SitesList.Items.Exchange(SitesList.ItemIndex, SitesList.ItemIndex+1);
  Modified := True;
  Caption := ExtractFileName(CurrentProjectName)+'* - Web Site Sizzler';
end;

procedure TMainForm.FormCloseQuery(Sender: TObject; var CanClose: Boolean);
begin
  if Modified then
   if Application.MessageBox('Would you like to save your work before exiting?', 'Confirm', MB_YESNO+MB_ICONQUESTION) = mrYes then
    SaveProject1Click(Self);
  CanClose := True;
end;

function CopyDir(const fromDir, toDir: string): Boolean;
var
  fos: TSHFileOpStruct;
begin
  ZeroMemory(@fos, SizeOf(fos));
  with fos do
  begin
    wFunc  := FO_COPY;
    fFlags := FOF_NOCONFIRMATION or FOF_SILENT{ or FOF_FILESONLY};
    pFrom  := PChar(fromDir + #0);
    pTo    := PChar(toDir)
  end;
  Result := (0 = ShFileOperation(fos));
end;


function MoveDir(const fromDir, toDir: string): Boolean;
var
  fos: TSHFileOpStruct;
begin
  ZeroMemory(@fos, SizeOf(fos));
  with fos do
  begin
    wFunc  := FO_MOVE;
    fFlags := FOF_NOCONFIRMATION or FOF_SILENT{ or FOF_FILESONLY};
    pFrom  := PChar(fromDir + #0);
    pTo    := PChar(toDir)
  end;
  Result := (0 = ShFileOperation(fos));
end;

function DelDir(dir: string): Boolean;
var
  fos: TSHFileOpStruct;
begin
  ZeroMemory(@fos, SizeOf(fos));
  with fos do
  begin
    wFunc  := FO_DELETE;
    fFlags := FOF_SILENT or FOF_NOCONFIRMATION;
    pFrom  := PChar(dir + #0);
  end;
  Result := (0 = ShFileOperation(fos));
end;

procedure TMainForm.ProcessAllWebsites1Click(Sender: TObject);
var I: Integer;
    Report: TReportForm;
    Result: String;
begin
  if NrOfSites <= 0 then Exit;

  Enabled := False;
  Report := TReportForm.Create(Self);
  with TProcessingForm.Create(Self) do
  begin
  try
   PBar.Max := NrOfSites;
   PBar.Position := 0;
   Show;
   for I:=0 to NrOfSites-1 do
   begin
     lblCurrentSite.Caption := Sites[I].Settings.WebsiteName;
     PBar.Position := I+1;
     Application.ProcessMessages;
     // perform upload
     lblOperation.Caption := 'Preparing to build website...';
     Application.ProcessMessages;
     CopyRSSGMDirToTempDir;
     // change CONFIG.PHP
     lblOperation.Caption := 'Applying configuration of current site...';
     Application.ProcessMessages;
     ApplySiteConfiguration(I);
     // keywords?
     lblOperation.Caption := 'Configuring keywords...';
     Application.ProcessMessages;
     ConfigureKeywords(I);
     // articles?
     lblOperation.Caption := 'Configuring articles...';
     Application.ProcessMessages;
     ConfigureArticles(I);
     // upload site
     lblOperation.Caption := 'Uploading website...';
     Application.ProcessMessages;
     UploadSite(I);
     // auto-generate articles?
     if Sites[I].Keyword.AutoGenerateArticles then
      AutoGenerateArticles(I);
     // finish up by running the generator
     lblOperation.Caption := 'Building website...';
     Application.ProcessMessages;
     Result := BuildSite(I);
     if Pos('Done!', Result) > 0 then
      Report.ListBox.Items.Add(Sites[I].Settings.URL+'~1')
     else
      Report.ListBox.Items.AddObject(Sites[I].Settings.URL+'~0xFFFF~'+Result, TObject(Result));
   end;
   // delete traces
   DelDir(ExtractFilePath(ParamStr(0))+'temp');
  except
   on E: Exception do ShowMessage(E.Message);
  end;
  // delete traces
  DelDir(ExtractFilePath(ParamStr(0))+'temp');
  Free;
  end;
  Report.ShowModal;
  Report.Free;
  Enabled := True;
end;

procedure TMainForm.CopyRSSGMDirToTempDir;
begin
  ForceDirectories(ExtractFilePath(PAramStr(0))+'temp');
  CopyDir(ExtractFilePath(PAramStr(0))+'rssgm', ExtractFilePath(PAramStr(0))+'temp');
end;

procedure TMainForm.ApplySiteConfiguration(Index: Integer);
var List: TStringList;
begin
  List := TStringList.Create;
  List.LoadFromFile(ExtractFilePath(PAramStr(0))+'temp\rssgm\config.php');
  List.Text := StringReplace(List.Text, '%mysitename%', Sites[Index].Settings.WebsiteName, [rfReplaceAll, rfIgnoreCase]);
  List.Text := StringReplace(List.Text, '%mainkeyword%', Sites[Index].Settings.MainKeyword, [rfReplaceAll, rfIgnoreCase]);
  List.Text := StringReplace(List.Text, '%adsense_pub%', Sites[Index].Settings.AdsensePub, [rfReplaceAll, rfIgnoreCase]);
  List.Text := StringReplace(List.Text, '%chitika_pub%', Sites[Index].Settings.ChitikaPub, [rfReplaceAll, rfIgnoreCase]);
  List.Text := StringReplace(List.Text, '%chitika_channel%', Sites[Index].Settings.ChitikaChannel, [rfReplaceAll, rfIgnoreCase]);

  List.Text := StringReplace(List.Text, '%_aff%', Sites[Index].Settings.PeakClickAffID, [rfReplaceAll, rfIgnoreCase]);
  List.Text := StringReplace(List.Text, '%_subaff%', Sites[Index].Settings.PeackClickSubaffID, [rfReplaceAll, rfIgnoreCase]);
  List.Text := StringReplace(List.Text, '%_numResults%', Sites[Index].Settings.NrOfSearchResults, [rfReplaceAll, rfIgnoreCase]);
  if Sites[Index].Settings.DisableThumbs then
   List.Text := StringReplace(List.Text, '%_thumbs%', '0', [rfReplaceAll, rfIgnoreCase])
  else
   List.Text := StringReplace(List.Text, '%_thumbs%', '1', [rfReplaceAll, rfIgnoreCase]);
  List.SaveToFile(ExtractFilePath(PAramStr(0))+'temp\rssgm\config.php');
  List.Free;
end;

procedure TMainForm.ConfigureKeywords(Index: Integer);
begin
  if Sites[Index].Keyword.UseOwnKeywords then
  begin
    // copy the keywords file to the destination folder
    CopyDir(Sites[Index].Keyword.KeywordFile, ExtractFilePath(PAramStr(0))+'temp\rssgm\keywords\'+ExtractFileName(Sites[Index].Keyword.KeywordFile));
  end
  else
  if Sites[Index].Keyword.AutoScrapeKeyword then
  begin
    // auto-scrape keywords
    AutoScrapeKeywords(Index, ExtractFilePath(ParamStr(0))+'temp\rssgm\keywords\Keywords.txt');
  end;
end;

function AddSlash(S: String): String;
begin
  if S <> '' then
   if S[Length(S)] <> '\' then
    S := S + '\';
  Result := S;
end;

procedure TMainForm.ConfigureArticles(Index: Integer);
begin
  if Sites[Index].Keyword.UseOwnArticles then
  begin
    // copy the keywords file to the destination folder
    CopyDir(AddSlash(Sites[Index].Keyword.ArticleFolder)+'*.*', ExtractFilePath(ParamStr(0))+'temp\rssgm\articles\');
  end;
end;

procedure TMainForm.AssignFTPProperties(Index: Integer);
begin
  with FTP do
  begin
    Host := Sites[Index].FTP.Hostname;
    try
     Port := StrToInt(Sites[Index].FTP.Port);
    except
     Port := 21;
    end;
    Username := Sites[Index].FTP.UserName;
    Password := Sites[Index].FTP.Password;
    Passive := Sites[Index].FTP.PassiveTransfer;

    if not Sites[Index].FTP.UseProxy then
     ProxySettings.ProxyType := fpcmNone
    else
     case Sites[Index].FTP.ProxyType of
      0: ProxySettings.ProxyType := fpcmHTTPProxyWithFTP;
      1: ProxySettings.ProxyType := fpcmOpen;
      2: ProxySettings.ProxyType := fpcmSite;
      3: ProxySettings.ProxyType := fpcmTransparent;
      4: ProxySettings.ProxyType := fpcmUserPass;
      5: ProxySettings.ProxyType := fpcmUserSite;
     end; { case }

    ProxySettings.Host := Sites[Index].FTP.ProxyHostname;
    ProxySettings.UserName := Sites[Index].FTP.ProxyUsername;
    ProxySettings.Password := Sites[Index].FTP.ProxyPassword;
    try
     ProxySettings.Port := StrToInt(Sites[Index].FTP.ProxyPort);
    except
     ProxySettings.Port := 0;
    end;
  end;
end;

procedure TMainForm.UploadSite(Index: Integer);

function Add(S: String): String;
begin
  if S<>'' then
   if S[Length(S)]<>'\' then S := S + '\';
  Result := S;
end;

procedure MakeFTPDir(S: String);
begin
  try
   FTP.MakeDir(S);
  except
  end;
end;

procedure ChangeFTPDir(S: String);
begin
  FTP.ChangeDir(S);
end;

procedure UploadFile(Name: String);
begin
  if FTP.Connected then
  begin
    FTP.Put(Name, ExtractFileName(Name));
    ChangeFTPDir(FTP.RetrieveCurrentDir);
  end;
end;

procedure ProcessDir(S: String);
var
  sr: TSearchRec;
  I, FileAttrs: Integer;
  DirList: TStringList;
begin
  DirList := TStringList.Create;

  MakeFTPDir(ExtractFileName(S));
  ChangeFTPDir(ExtractFilename(S));
  S := S + '\';

  // process directory
  FileAttrs := faArchive + faDirectory;
  if FindFirst(S+'*.*', FileAttrs, sr) = 0 then
    begin
      repeat
        if FileExists(S+sr.Name) then
          // upload this file
          UploadFile(S+sr.Name)
        else
        if DirectoryExists(S+sr.Name) and (sr.Name <> '.') and (sr.Name <> '..') then
         DirList.Add(S+sr.Name);
      until FindNext(sr) <> 0;
      FindClose(sr);
    end;

  for I:=0 to DirList.Count-1 do
   ProcessDir(DirList[I]);

  ChangeFTPDir('..');
  DirList.Free;
end;

var DirName: String;
begin
  AssignFTPProperties(Index);
  try
   FTP.Connect();
   DirName := StringReplace(Sites[Index].FTP.Directory, '\', '/', [rfReplaceAll]);
   if DirName <> '' then
    FTP.ChangeDir(DirName);

   // ************** now upload the files ***************
   ProcessDir(ExtractFilePath(ParamStr(0))+'temp\rssgm');
   // **************** end upload files *****************

   // setup file permissions
   if not (FTP.RetrieveCurrentDir = 'rssgm') then
   begin
     FTP.SendCmd('chmod 755 rssgm'); // set RSSGM to 755
     FTP.ChangeDir('rssgm'); // go into the RSSGM folder
   end
   else
   begin
     FTP.ChangeDir('..');
     FTP.SendCmd('chmod 755 rssgm'); // set RSSGM to 755
     FTP.ChangeDir('rssgm');
   end;
   FTP.SendCmd('chmod 755 articles'); // set RSSGM/ARTICLES to 755

   if FTP.RetrieveCurrentDir = 'rssgm' then // get back
    FTP.ChangeDir('..');

   // we are now in RSSGM folder
   try
     FTP.MakeDir('cache');
   except
   end;
   FTP.SendCmd('chmod 755 cache');

   // chmod stuff
   FTP.ChangeDir('/');
   if Sites[Index].FTP.Directory <> '' then
    FTP.SendCmd('chmod 755 '+Sites[Index].FTP.Directory);

   // disconnect
   FTP.Disconnect();
  except
   on E: Exception do
   begin
     if FTP.Connected then
     try
       FTP.Disconnect();
     except
     end;
     Application.MessageBox(PChar('The connection could not be established due to the following error: '+#13#10#13#10+E.Message), 'Warning', MB_OK + MB_ICONWARNING);
   end;
  end;
end;

procedure TMainForm.AutoGenerateArticles(Index: Integer);
function URLEncode(const S: string; const InQueryString: Boolean): string;
var
  Idx: Integer; // loops thru characters in string
begin
  Result := '';
  for Idx := 1 to Length(S) do
  begin
    case S[Idx] of
      'A'..'Z', 'a'..'z', '0'..'9', '-', '_', '.':
        Result := Result + S[Idx];
      ' ':
        if InQueryString then
          Result := Result + '+'
        else
          Result := Result + '%20';
      else
        Result := Result + '%' + SysUtils.IntToHex(Ord(S[Idx]), 2);
    end;
  end;
end;
function AddSlash(S: String): String;
begin
  if S <> '' then
   if S[Length(S)] <> '/' then
    S := S + '/';
  Result := S;
end;
var
  List, params: TStringList;
  response: string;
  aStream: TMemoryStream;
//  Params: TStringStream;
begin
  List := TStringList.Create;
  params:= TStringList.Create;

  if Sites[Index].Keyword.UseOwnKeywords then
   List.LoadFromFile(Sites[Index].Keyword.KeywordFile)
  else
   // load from list of scraped keywords
   List.LoadFromFile(ExtractFilePath(ParamStr(0))+'temp\rssgm\keywords\Keywords.txt');

  try
    params.Values['keywords'] :=List.Text; // the list of keywords
    params.Values['artnum']:= Sites[Index].Keyword.ArticlesPerKeyword; // nr. of articles per keyword
    params.Values['numwords']:= Sites[Index].Keyword.WordsPerArticle; // number of words per article
    params.Values['gran']:= Sites[Index].Keyword.MarkovGranularity; // Markov granularity
    params.Values['submit']:= 'Do it'; // nr. of articles per keyword

    response:= HTTP.Post(AddSlash(Sites[Index].Settings.URL)+'rssgm/wiki.php ', params);
{  aStream := TMemoryStream.create;
  Params := TStringStream.create('');

  try
    with HTTP do
    begin
      Params.WriteString(URLEncode('keywords=' + List.Text, True) + '&');
      Params.WriteString(URLEncode('artnum=' + Sites[Index].Keyword.ArticlesPerKeyword, True) + '&');
      Params.WriteString(URLEncode('numwords=' + Sites[Index].Keyword.WordsPerArticle, True) + '&');
      Params.WriteString(URLEncode('gran=' + Sites[Index].Keyword.MarkovGranularity, True));
      Request.ContentType := 'application/x-www-form-urlencoded';
      try
      showmessage('before post');
        Post(AddSlash(Sites[Index].Settings.URL)+'rssgm/article_gen.php', Params, aStream);
      showmessage('after post');
      except
        on E: Exception do
          showmessage('Error encountered during POST: ' + E.Message);
      end;
    end;
  aStream.WriteBuffer(#0' ', 1);
  aStream.Position := 0;
//  Memo1.Lines.LoadFromStream(aStream);
  except
  end;
 }
//    result:= Pos('success', response);
  finally
    params.Free;
    List.Free;
  end;
end;

function TMainForm.BuildSite(Index: Integer): String;
function AddSlash(S: String): String;
begin
  if S <> '' then
   if S[Length(S)] <> '/' then
    S := S + '/';
  Result := S;
end;
var
  params: TStringList;
begin
  params:= TStringList.Create;
  try
    Result := HTTP.Post(AddSlash(Sites[Index].Settings.URL)+'rssgm/generator.php', params);
  finally
    params.Free;
  end;
end;

procedure TMainForm.AutoScrapeKeywords(Index: Integer; SaveToFilename: String);

function FillList(sentnc    : WideString;      {Input WideString}
                  var sList : TStringList; {WideString List to add values to}
                  clearList : Boolean)     {Clear list before adding?}
                  : Boolean;               {Return value}
var
  str,
  wrd     : WideString;
  I       : Word;
begin


  {Initialize vars}
  Result := True;
  str := sentnc;
  wrd := '';

  {Check to see if the WideString passed is blank}
  if (Length(sentnc) = 0) then
  begin
    Result := False;
    Exit;
  end;

  {Clear the list if wanted and the count of values is > 0}
  if clearList AND (sList.Count > 0) then
    repeat
      sList.Delete(0);
    until
      sList.Count = 0;

  while (Pos(' ', str) > 0) do                  {Do this while you find}
  begin                                         {spaces in the sentence}
    wrd := Copy(str, 1, Pos(' ', str) - 1);     {Get the word from the WideString}
    sList.Add(wrd);                             {Add the word to the TStringList}
    str := Copy(str, Pos(' ', str) + 1,         {Redefine the sentence by cutting}
                Length(str) - Length(wrd) + 1); {off the first word}
  end;

  if (Length(str) > 0) then                     {This is important, because you never}
    sList.Add(str);                             {know if there's anything left in the
                                                 sentence.}
end;

var MyList, Words: TStringList;
    regexp: TRegExpr;
    http: TIdHTTP;
    wordthreadlist: TList;
    searchstr, RetrievedTXT, s: String;
    MaxWords: Integer;
    AddToList: Boolean;
begin
    //search overture suggestion first
    MyList := TStringList.Create;

    regexp := TRegExpr.Create;
    http := TIdHttp.Create(nil);
    wordthreadlist := TList.Create;

    http.Request.UserAgent := 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)';
    s := 'http://inventory.overture.com/d/searchinventory/suggestion/?mkt=us&lang=en-us&term=' + AnsiReplaceText(Sites[Index].Keyword.StartingKeyword, ' ', '+');
    RetrievedTXT := http.Get(s);
    s :=  RetrievedTXT;

    regexp.Expression := '(\d+)</td>\s*<[^>]+><[^>]+>([^<]+)</a></td>';
    if regexp.Exec(s) then
    begin
      repeat
        searchstr := AnsiReplaceText(regexp.Match[2], '&nbsp;', '');

        if searchstr <> '' then
        begin
          // check to see if we have the specified number of max. words to look for
          try
            MaxWords := StrToInt(Sites[Index].Keyword.MaxNrOfKeywords);
          except
            MaxWords := 1000;
          end;
          AddToList := False;
          if MaxWords > 0 then
          begin
            Words := TStringList.Create;
            FillList(searchstr, Words, True);
            if Words.Count <= MaxWords then
             AddToList := True; 
            Words.Free;
          end
          else
           AddToList := True;

          if AddToList then
          begin
           MyList.Add(searchstr);
{           li := pSearch^.lvResults.Items.Add;
           li.Caption := searchstr;
           //li.SubItems.Add(regexp.Match[1]);
           pSearch^.strs.Add(searchstr);
           pSearch^.counts.Add(0);
           pSearch^.depths.Add(Pointer(depth+1));}
          end;
        end;

      until not regexp.ExecNext;
    end;

    regexp.Expression := '(\d+)</td>\s*<td>&nbsp;<a [^>]+><font [^>]+>([^<]+)</a></td>';
    if regexp.Exec(s) then
    begin
      repeat
        searchstr := AnsiReplaceText(regexp.Match[2], '&nbsp;', '');

        if searchstr <> '' then
        begin
          // check to see if we have the specified number of max. words to look for
          try
            MaxWords := StrToInt(Sites[Index].Keyword.MaxNrOfKeywords);
          except
            MaxWords := 1000;
          end;
          AddToList := False;
          if MaxWords > 0 then
          begin
            Words := TStringList.Create;
            FillList(searchstr, Words, True);
            if Words.Count <= MaxWords then
             AddToList := True; 
            Words.Free;
          end
          else
           AddToList := True;

          if AddToList then
          begin
            MyList.Add(searchstr);
{            li := pSearch^.lvResults.Items.Add;
            li.Caption := UTF8Decode(searchstr);
            li.SubItems.Add(regexp.Match[1]);
            pSearch^.strs.Add(searchstr);
            pSearch^.counts.Add(0);
            pSearch^.depths.Add(Pointer(depth+1));}
          end;
        end;

      until not regexp.ExecNext;
    end;

   MyList.SaveToFile(SaveToFilename);
   MyList.Free;
end;

function TMainForm.ExtractFilePath(FileName: String): String;
begin
  if FileName <> '' then
   if FileName[1] = '"' then
    Delete(FileName, 1, 1);
  if FileName <> '' then
   if FileName[Length(FileName)] = '"' then
    Delete(FileName, Length(FileName), 1);
  Result := SysUtils.ExtractFilePath(FileName);
end;

function TMainForm.ExtractFileName(FileName: String): String;
begin
  if FileName <> '' then
   if FileName[1] = '"' then
    Delete(FileName, 1, 1);
  if FileName <> '' then
   if FileName[Length(FileName)] = '"' then
    Delete(FileName, Length(FileName), 1);
  Result := SysUtils.ExtractFileName(FileName);
end;

end.
