First draft of web resource service; fetches data from a JSON feed

and stores it in user prefs, where it can be used by the new tab page.

BUG = http://crbug.com/13363

Review URL: http://codereview.chromium.org/125052

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@18766 0039d316-1c4b-4281-b951-d872f2087c98

chrome/app/generated_resources.grd

@@ -3409,6 +3409,10 @@ each locale. -->
                desc="Leading sentence above a custom logo that the theme artist has provided">
         Theme created by
       </message>
+      <message name="IDS_NEW_TAB_WEB_RESOURCE_CACHE"
+               desc="Title of block that holds tips, suggestions, and other new information gathered from web resources in the new tab page.">
+        Recommendations
+      </message>
 
       <!-- SafeBrowsing -->
       <message name="IDS_SAFE_BROWSING_MALWARE_TITLE" desc="SafeBrowsing Malware HTML title">

chrome/browser/browser_main.cc

@@ -110,7 +110,7 @@
 
 #if defined(LINUX2)
 #include "chrome/browser/extensions/extension_protocols.h"
-#endif // defined(LINUX2)
+#endif  // defined(LINUX2)
 
 #if defined(TOOLKIT_GTK)
 #include "chrome/common/gtk_util.h"
@@ -790,6 +790,10 @@ int BrowserMain(const MainFunctionParams& parameters) {
   RecordBreakpadStatusUMA(metrics);
   // Start up the extensions service. This should happen before Start().
   profile->InitExtensions();
+  // Start up the web resource service.  This starts loading data after a
+  // short delay so as not to interfere with startup time.
+  if (parsed_command_line.HasSwitch(switches::kWebResources))
+    profile->InitWebResources();
 
   int result_code = ResultCodes::NORMAL_EXIT;
   if (parameters.ui_task) {

chrome/browser/dom_ui/new_tab_ui.cc

@@ -22,6 +22,7 @@
 #include "chrome/browser/dom_ui/dom_ui_theme_source.h"
 #include "chrome/browser/dom_ui/downloads_dom_handler.h"
 #include "chrome/browser/dom_ui/history_ui.h"
+#include "chrome/browser/dom_ui/web_resource_handler.h"
 #include "chrome/browser/history/page_usage_data.h"
 #include "chrome/browser/metrics/user_metrics.h"
 #include "chrome/browser/profile.h"
@@ -291,6 +292,8 @@ void NewTabHTMLSource::StartDataRequest(const std::string& path,
       l10n_util::GetString(IDS_NEW_TAB_RECENTLY_CLOSED_WINDOW_MULTIPLE));
   localized_strings.SetString(L"attributionintro",
       l10n_util::GetString(IDS_NEW_TAB_ATTRIBUTION_INTRO));
+  localized_strings.SetString(L"resourcecache",
+      l10n_util::GetString(IDS_NEW_TAB_WEB_RESOURCE_CACHE));
 
   SetFontAndTextDirection(&localized_strings);
 
@@ -1380,6 +1383,9 @@ NewTabUI::NewTabUI(TabContents* contents)
       downloads_handler->Init();
     }
 
+    if (EnableWebResources())
+      AddMessageHandler(new WebResourceHandler(this));
+
     AddMessageHandler(new TemplateURLHandler(this));
     AddMessageHandler(new MostVisitedHandler(this));
     AddMessageHandler(new RecentlyBookmarkedHandler(this));
@@ -1435,6 +1441,8 @@ void NewTabUI::Observe(NotificationType type,
 // static
 void NewTabUI::RegisterUserPrefs(PrefService* prefs) {
   MostVisitedHandler::RegisterUserPrefs(prefs);
+  if (NewTabUI::EnableWebResources())
+    WebResourceHandler::RegisterUserPrefs(prefs);
 }
 
 // static
@@ -1442,3 +1450,9 @@ bool NewTabUI::EnableNewNewTabPage() {
   const CommandLine* command_line = CommandLine::ForCurrentProcess();
   return command_line->HasSwitch(switches::kNewNewTabPage);
 }
+
+bool NewTabUI::EnableWebResources() {
+  const CommandLine* command_line = CommandLine::ForCurrentProcess();
+  return command_line->HasSwitch(switches::kWebResources);
+}
+

chrome/browser/dom_ui/new_tab_ui.h

@@ -28,6 +28,9 @@ class NewTabUI : public DOMUI,
   // Whether we should use the prototype new tab page.
   static bool EnableNewNewTabPage();
 
+  // Whether we should enable the web resources backend service
+  static bool EnableWebResources();
+
  private:
   void Observe(NotificationType type,
                const NotificationSource& source,

chrome/browser/dom_ui/web_resource_handler.cc

@@ -0,0 +1,95 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/values.h"
+#include "chrome/browser/dom_ui/web_resource_handler.h"
+#include "chrome/browser/profile.h"
+#include "chrome/browser/web_resource/web_resource_service.h"
+#include "chrome/common/web_resource/web_resource_unpacker.h"
+#include "chrome/common/pref_names.h"
+
+namespace {
+
+  const int kNumWebResourcesToShow = 2;
+
+  // TODO(mrc): l10n
+  // This title should only appear the very first time Chrome is run with
+  // web resources enabled; otherwise the cache should be populated.
+  static const wchar_t* kWebResourceTitleAtStartup =
+      L"New: Suggestion Box!";
+
+  // This snipp should only appear the very first time Chrome is run with
+  // web resources enabled; otherwise the cache should be populated.
+  static const wchar_t* kWebResourceSnippetAtStartup =
+      L"Tips and recommendations to help you discover interesting websites.";
+}
+
+WebResourceHandler::WebResourceHandler(DOMUI* dom_ui)
+    : DOMMessageHandler(dom_ui),
+      dom_ui_(dom_ui) {
+  dom_ui->RegisterMessageCallback("getNextCachedWebResource",
+      NewCallback(this, &WebResourceHandler::HandleGetCachedWebResource));
+
+  web_resource_cache_ = dom_ui_->GetProfile()->GetPrefs()->
+      GetDictionary(prefs::kNTPWebResourceCache);
+}
+
+void WebResourceHandler::HandleGetCachedWebResource(const Value* content) {
+  // Eventually we will feed more than one web resource datum at a time
+  // to the NTP; for now, this is a list containing one item: the tip
+  // to be displayed.
+  ListValue list_value;
+
+  // Holds the web resource data found in the preferences cache.
+  DictionaryValue* wr_dict;
+
+  // Dictionary which will be sent back in a Javascript call.
+  DictionaryValue* tip_dict = new DictionaryValue();
+
+  // These values hold the data for each web resource item.  As the web
+  // resource server solidifies, these may change.
+  std::wstring title;
+  std::wstring thumb;
+  std::wstring source;
+  std::wstring snipp;
+  std::wstring url;
+
+  // This should only be true on the very first Chrome run; otherwise,
+  // the cache should be populated.
+  if (web_resource_cache_ == NULL || web_resource_cache_->GetSize() < 1) {
+    title = kWebResourceTitleAtStartup;
+    snipp = kWebResourceSnippetAtStartup;
+  } else {
+    // Right now, hard-coded to simply get the first item (marked "0") in the
+    // resource data stored in the cache.  Fail silently if data is missing.
+    // TODO(mrc): If data is missing, iterate through cache.
+    web_resource_cache_->GetDictionary(L"0", &wr_dict);
+    if (wr_dict &&
+        wr_dict->GetSize() > 0 &&
+        wr_dict->GetString(WebResourceService::kWebResourceTitle, &title) &&
+        wr_dict->GetString(WebResourceService::kWebResourceThumb, &thumb) &&
+        wr_dict->GetString(WebResourceService::kWebResourceSource, &source) &&
+        wr_dict->GetString(WebResourceService::kWebResourceSnippet, &snipp) &&
+        wr_dict->GetString(WebResourceService::kWebResourceURL, &url)) {
+      tip_dict->SetString(WebResourceService::kWebResourceTitle, title);
+      tip_dict->SetString(WebResourceService::kWebResourceThumb, thumb);
+      tip_dict->SetString(WebResourceService::kWebResourceSource, source);
+      tip_dict->SetString(WebResourceService::kWebResourceSnippet, snipp);
+      tip_dict->SetString(WebResourceService::kWebResourceURL, url);
+    }
+  }
+
+  list_value.Append(tip_dict);
+
+  // Send list of snippets back out to the DOM.
+  dom_ui_->CallJavascriptFunction(L"nextWebResource", list_value);
+}
+
+// static
+void WebResourceHandler::RegisterUserPrefs(PrefService* prefs) {
+  prefs->RegisterDictionaryPref(prefs::kNTPWebResourceCache);
+  prefs->RegisterStringPref(prefs::kNTPWebResourceServer,
+      WebResourceService::kDefaultResourceServer);
+}
+

chrome/browser/dom_ui/web_resource_handler.h

@@ -0,0 +1,56 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This class pulls data from a web resource (such as a JSON feed) which
+// has been stored in the user's preferences file.  Used mainly
+// by the suggestions and tips area of the new tab page.
+
+// Current sketch of tip cache format, hardcoded for poptart data in
+// basic text form:
+
+// "web_resource_cache": {
+//    "0": {
+//        "index": should become time field (or not)
+//        "snippet": the text of the item
+//        "source": text describing source (i.e., "New York Post")
+//        "thumbnail": URL of thumbnail on popgadget server
+//        "title": text giving title of item
+//        "url": link to item's page
+//    },
+//    [up to number of items in kMaxWebResourceCacheSize]
+
+#ifndef CHROME_BROWSER_DOM_UI_WEB_RESOURCE_HANDLER_H_
+#define CHROME_BROWSER_DOM_UI_WEB_RESOURCE_HANDLER_H_
+
+#include "chrome/browser/dom_ui/dom_ui.h"
+
+class DictionaryValue;
+class DOMUI;
+class PrefService;
+class Value;
+
+class WebResourceHandler : public DOMMessageHandler {
+ public:
+  explicit WebResourceHandler(DOMUI* dom_ui);
+
+  WebResourceHandler();
+
+  // Callback which pulls web resource data from the preferences.
+  void HandleGetCachedWebResource(const Value* content);
+
+  // Register web resource cache with pref service.
+  static void RegisterUserPrefs(PrefService* prefs);
+
+ private:
+  // So we can push data out to the page that has called this handler.
+  DOMUI* dom_ui_;
+
+  // Filled with data from cache in preferences.
+  const DictionaryValue* web_resource_cache_;
+
+  DISALLOW_COPY_AND_ASSIGN(WebResourceHandler);
+};
+
+#endif  // CHROME_BROWSER_DOM_UI_WEB_RESOURCE_HANDLER_H_
+

chrome/browser/profile.cc

@@ -353,6 +353,10 @@ class OffTheRecordProfileImpl : public Profile,
     NOTREACHED();
   }
 
+  virtual void InitWebResources() {
+    NOTREACHED();
+  }
+
   virtual void ExitedOffTheRecordMode() {
     // Drop our download manager so we forget about all the downloads made
     // in off-the-record mode.
@@ -480,6 +484,14 @@ void ProfileImpl::InitExtensions() {
   extensions_service_->Init();
 }
 
+void ProfileImpl::InitWebResources() {
+  web_resource_service_ = new WebResourceService(
+      this,
+      g_browser_process->file_thread()->message_loop());
+
+  web_resource_service_->StartAfterDelay();
+}
+
 ProfileImpl::~ProfileImpl() {
   tab_restore_service_ = NULL;
 

chrome/browser/profile.h

@@ -15,6 +15,7 @@
 #include "base/file_path.h"
 #include "base/scoped_ptr.h"
 #include "base/timer.h"
+#include "chrome/browser/web_resource/web_resource_service.h"
 #ifdef CHROME_PERSONALIZATION
 #include "chrome/personalization/personalization.h"
 #endif
@@ -277,6 +278,9 @@ class Profile {
 
   virtual void InitExtensions() = 0;
 
+  // Start up service that gathers data from web resource feeds.
+  virtual void InitWebResources() = 0;
+
 #ifdef UNIT_TEST
   // Use with caution.  GetDefaultRequestContext may be called on any thread!
   static void set_default_request_context(URLRequestContext* c) {
@@ -352,6 +356,7 @@ class ProfileImpl : public Profile,
   virtual SpellChecker* GetSpellChecker();
   virtual void MarkAsCleanShutdown();
   virtual void InitExtensions();
+  virtual void InitWebResources();
 #ifdef CHROME_PERSONALIZATION
   virtual ProfilePersonalization* GetProfilePersonalization();
 #endif
@@ -399,6 +404,7 @@ class ProfileImpl : public Profile,
   scoped_ptr<TemplateURLFetcher> template_url_fetcher_;
   scoped_ptr<TemplateURLModel> template_url_model_;
   scoped_ptr<BookmarkModel> bookmark_bar_model_;
+  scoped_refptr<WebResourceService> web_resource_service_;
 
 #ifdef CHROME_PERSONALIZATION
   scoped_ptr<ProfilePersonalization> personalization_;

chrome/browser/utility_process_host.cc

@@ -37,6 +37,14 @@ bool UtilityProcessHost::StartExtensionUnpacker(const FilePath& extension) {
   return true;
 }
 
+bool UtilityProcessHost::StartWebResourceUnpacker(const std::string& data) {
+  if (!StartProcess(FilePath()))
+    return false;
+
+  Send(new UtilityMsg_UnpackWebResource(data));
+  return true;
+}
+
 bool UtilityProcessHost::StartProcess(const FilePath& exposed_dir) {
   if (!CreateChannel())
     return false;
@@ -56,7 +64,10 @@ bool UtilityProcessHost::StartProcess(const FilePath& exposed_dir) {
 
   base::ProcessHandle process;
 #if defined(OS_WIN)
-  process = sandbox::StartProcessWithAccess(&cmd_line, exposed_dir);
+  if (exposed_dir.empty())
+    process = sandbox::StartProcess(&cmd_line);
+  else
+    process = sandbox::StartProcessWithAccess(&cmd_line, exposed_dir);
 #else
   // TODO(port): sandbox
   base::LaunchApp(cmd_line, false, false, &process);
@@ -89,5 +100,9 @@ void UtilityProcessHost::Client::OnMessageReceived(
                         Client::OnUnpackExtensionSucceeded)
     IPC_MESSAGE_HANDLER(UtilityHostMsg_UnpackExtension_Failed,
                         Client::OnUnpackExtensionFailed)
+    IPC_MESSAGE_HANDLER(UtilityHostMsg_UnpackWebResource_Succeeded,
+                        Client::OnUnpackWebResourceSucceeded)
+    IPC_MESSAGE_HANDLER(UtilityHostMsg_UnpackWebResource_Failed,
+                        Client::OnUnpackWebResourceFailed)
   IPC_END_MESSAGE_MAP_EX()
 }

chrome/browser/utility_process_host.h

@@ -15,6 +15,7 @@
 
 class CommandLine;
 class DictionaryValue;
+class ListValue;
 class MessageLoop;
 
 // This class acts as the browser-side host to a utility child process.  A
@@ -42,6 +43,17 @@ class UtilityProcessHost : public ChildProcessHost {
     // |error_message| contains a description of the problem.
     virtual void OnUnpackExtensionFailed(const std::string& error_message) {}
 
+    // Called when the web resource has been successfully parsed.  |json_data|
+    // contains the parsed list of web resource items downloaded from the
+    // web resource server.
+    virtual void OnUnpackWebResourceSucceeded(
+        const ListValue& json_data) {}
+
+    // Called when an error occurred while parsing the resource data.
+    // |error_message| contains a description of the problem.
+    virtual void OnUnpackWebResourceFailed(
+        const std::string& error_message) {}
+
    private:
     friend class UtilityProcessHost;
     void OnMessageReceived(const IPC::Message& message);
@@ -59,8 +71,16 @@ class UtilityProcessHost : public ChildProcessHost {
   // location first.
   bool StartExtensionUnpacker(const FilePath& extension);
 
+  // Start a process to unpack and parse a web resource from the given JSON
+  // data.  Any links that need to be downloaded from the parsed data
+  // (thumbnails, etc.) will be unpacked in resource_dir.
+  // TODO(mrc): Right now, the unpacker just parses the JSON data, and
+  // doesn't do any unpacking.  This should change once we finalize the
+  // web resource server format(s).
+  bool StartWebResourceUnpacker(const std::string& data);
+
  private:
-  // Starts the process.  Returns true iff it succeeded.
+  // Starts a process.  Returns true iff it succeeded.
   bool StartProcess(const FilePath& exposed_dir);
 
   // IPC messages:

chrome/browser/views/frame/browser_view.cc

@@ -837,7 +837,8 @@ void BrowserView::SetDownloadShelfVisible(bool visible) {
       GetDownloadShelf();
     }
 
-    browser_->UpdateDownloadShelfVisibility(visible);
+    if (browser_ != NULL)
+      browser_->UpdateDownloadShelfVisibility(visible);
   }
 
   // SetDownloadShelfVisible can force-close the shelf, so make sure we lay out
@@ -1289,7 +1290,7 @@ int BrowserView::NonClientHitTest(const gfx::Point& point) {
 }
 
 gfx::Size BrowserView::GetMinimumSize() {
-  // TODO: In theory the tabstrip width should probably be
+  // TODO(noname): In theory the tabstrip width should probably be
   // (OTR + tabstrip + caption buttons) width.
   gfx::Size tabstrip_size(
       browser_->SupportsWindowFeature(Browser::FEATURE_TABSTRIP) ?
@@ -1367,7 +1368,8 @@ void BrowserView::Init() {
 #if defined(OS_WIN)
   SetProp(GetWidget()->GetNativeView(), kBrowserViewKey, this);
 #else
-  g_object_set_data(G_OBJECT(GetWidget()->GetNativeView()), kBrowserViewKey, this);
+  g_object_set_data(G_OBJECT(GetWidget()->GetNativeView()),
+                    kBrowserViewKey, this);
 #endif
 
   // Start a hung plugin window detector for this browser object (as long as

chrome/browser/web_resource/web_resource_service.cc

@@ -0,0 +1,268 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+#include "chrome/browser/web_resource/web_resource_service.h"
+
+#include "base/string_util.h"
+#include "base/time.h"
+#include "base/values.h"
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/chrome_thread.h"
+#include "chrome/browser/profile.h"
+#include "chrome/browser/net/url_fetcher.h"
+#include "chrome/common/pref_names.h"
+#include "googleurl/src/gurl.h"
+#include "net/base/load_flags.h"
+#include "net/url_request/url_request_status.h"
+
+const wchar_t* WebResourceService::kWebResourceTitle = L"title";
+const wchar_t* WebResourceService::kWebResourceThumb = L"thumbnail";
+const wchar_t* WebResourceService::kWebResourceSource = L"source";
+const wchar_t* WebResourceService::kWebResourceSnippet = L"snippet";
+const wchar_t* WebResourceService::kWebResourceURL = L"url";
+
+class WebResourceService::WebResourceFetcher
+    : public URLFetcher::Delegate {
+ public:
+  explicit WebResourceFetcher(WebResourceService* web_resource_service) :
+      ALLOW_THIS_IN_INITIALIZER_LIST(fetcher_factory_(this)),
+      web_resource_service_(web_resource_service) {
+  }
+
+  // Delay initial load of resource data into cache so as not to interfere
+  // with startup time.
+  void StartAfterDelay(int delay_ms) {
+    MessageLoop::current()->PostDelayedTask(FROM_HERE,
+      fetcher_factory_.NewRunnableMethod(&WebResourceFetcher::StartFetch),
+                                         delay_ms);
+  }
+
+  // Initializes the fetching of data from the resource server.  Data
+  // load calls OnURLFetchComplete.
+  void StartFetch() {
+    // First, put our next cache load on the MessageLoop.
+    MessageLoop::current()->PostDelayedTask(FROM_HERE,
+      fetcher_factory_.NewRunnableMethod(&WebResourceFetcher::StartFetch),
+                                         kCacheUpdateDelay);
+    // If we are still fetching data, exit.
+    if (web_resource_service_->in_fetch_)
+      return;
+
+    url_fetcher_.reset(new URLFetcher(GURL(
+        WideToUTF8(web_resource_service_->web_resource_server_)),
+        URLFetcher::GET, this));
+    // Do not let url fetcher affect existing state in profile (by setting
+    // cookies, for example.
+    url_fetcher_->set_load_flags(net::LOAD_DISABLE_CACHE |
+      net::LOAD_DO_NOT_SAVE_COOKIES);
+    url_fetcher_->set_request_context(Profile::GetDefaultRequestContext());
+    url_fetcher_->Start();
+  }
+
+  // From URLFetcher::Delegate.
+  void OnURLFetchComplete(const URLFetcher* source,
+                          const GURL& url,
+                          const URLRequestStatus& status,
+                          int response_code,
+                          const ResponseCookies& cookies,
+                          const std::string& data) {
+    // Delete the URLFetcher when this function exits.
+    scoped_ptr<URLFetcher> clean_up_fetcher(url_fetcher_.release());
+
+    // Don't parse data if attempt to download was unsuccessful.
+    // Stop loading new web resource data, and silently exit.
+    if (!status.is_success() || (response_code != 200))
+      return;
+
+    web_resource_service_->UpdateResourceCache(data);
+  }
+
+ private:
+  // So that we can delay our start so as not to affect start-up time; also,
+  // so that we can schedule future cache updates.
+  ScopedRunnableMethodFactory<WebResourceFetcher> fetcher_factory_;
+
+  // The tool that fetches the url data from the server.
+  scoped_ptr<URLFetcher> url_fetcher_;
+
+  // Our owner and creator.
+  scoped_ptr<WebResourceService> web_resource_service_;
+};
+
+// This class coordinates a web resource unpack and parse task which is run in
+// a separate process.  Results are sent back to this class and routed to
+// the WebResourceService.
+class WebResourceService::UnpackerClient
+    : public UtilityProcessHost::Client {
+ public:
+  UnpackerClient(WebResourceService* web_resource_service,
+                 const std::string& json_data)
+    : web_resource_service_(web_resource_service),
+      json_data_(json_data) {
+  }
+
+  void Start() {
+    AddRef();  // balanced in Cleanup.
+
+    if (web_resource_service_->resource_dispatcher_host_) {
+      ChromeThread::GetMessageLoop(ChromeThread::IO)->PostTask(FROM_HERE,
+          NewRunnableMethod(this, &UnpackerClient::StartProcessOnIOThread,
+                            web_resource_service_->resource_dispatcher_host_,
+                            MessageLoop::current()));
+    } else {
+      // TODO(mrc): unit tests here.
+    }
+  }
+
+ private:
+  virtual void OnUnpackWebResourceSucceeded(const ListValue& parsed_json) {
+    web_resource_service_->OnWebResourceUnpacked(parsed_json);
+    Release();
+  }
+
+  virtual void OnUnpackWebResourceFailed(const std::string& error_message) {
+    web_resource_service_->EndFetch();
+    Release();
+  }
+
+  void StartProcessOnIOThread(ResourceDispatcherHost* rdh,
+                              MessageLoop* file_loop) {
+    UtilityProcessHost* host = new UtilityProcessHost(rdh, this, file_loop);
+    // TODO(mrc): get proper file path when we start using web resources
+    // that need to be unpacked.
+    host->StartWebResourceUnpacker(json_data_);
+  }
+
+  scoped_refptr<WebResourceService> web_resource_service_;
+
+  // Holds raw JSON string.
+  const std::string& json_data_;
+};
+
+const wchar_t* WebResourceService::kDefaultResourceServer =
+//    L"http://hokiepokie.nyc.corp.google.com:8125/"
+//    L"labs/popgadget/world?view=json";
+    L"http://sites.google.com/a/chromium.org/dev/developers/"
+    L"design-documents/web_resources/popgadget_test.json";
+
+const char* WebResourceService::kResourceDirectoryName =
+    "Resources";
+
+WebResourceService::WebResourceService(Profile* profile,
+                                       MessageLoop* backend_loop) :
+    prefs_(profile->GetPrefs()),
+    web_resource_dir_(profile->GetPath().AppendASCII(kResourceDirectoryName)),
+    backend_loop_(backend_loop),
+    in_fetch_(false) {
+  Init();
+}
+
+WebResourceService::~WebResourceService() { }
+
+void WebResourceService::Init() {
+  resource_dispatcher_host_ = g_browser_process->resource_dispatcher_host();
+  web_resource_fetcher_ = new WebResourceFetcher(this);
+  prefs_->RegisterStringPref(prefs::kNTPWebResourceCacheUpdate, L"0");
+  // TODO(mrc): make sure server name is valid.
+  web_resource_server_ = prefs_->HasPrefPath(prefs::kNTPWebResourceServer) ?
+      prefs_->GetString(prefs::kNTPWebResourceServer) :
+      kDefaultResourceServer;
+}
+
+void WebResourceService::EndFetch() {
+  in_fetch_ = false;
+}
+
+void WebResourceService::OnWebResourceUnpacked(const ListValue& parsed_json) {
+  // Get dictionary of cached preferences.
+  web_resource_cache_ =
+      prefs_->GetMutableDictionary(prefs::kNTPWebResourceCache);
+  ListValue::const_iterator wr_iter = parsed_json.begin();
+  int wr_counter = 0;
+
+  // These values store the data for each new web resource item.
+  std::wstring result_snippet;
+  std::wstring result_url;
+  std::wstring result_source;
+  std::wstring result_title;
+  std::wstring result_title_type;
+  std::wstring result_thumbnail;
+
+  // Iterate through newly parsed preferences, replacing stale cache with
+  // new data.
+  // TODO(mrc): make this smarter, so it actually only replaces stale data,
+  // instead of overwriting the whole thing every time.
+  while (wr_iter != parsed_json.end() &&
+         wr_counter < kMaxResourceCacheSize) {
+    // Each item is stored in the form of a dictionary.
+    // See web_resource_handler.h for format (this will change until
+    // web resource services are solidified!).
+    if (!(*wr_iter)->IsType(Value::TYPE_DICTIONARY))
+      continue;
+    DictionaryValue* wr_dict =
+        static_cast<DictionaryValue*>(*wr_iter);
+
+    // Get next space for resource in prefs file.
+    Value* current_wr;
+    std::wstring wr_counter_str = IntToWString(wr_counter);
+    // Create space if it doesn't exist yet.
+    if (!web_resource_cache_->Get(wr_counter_str, &current_wr) ||
+      !current_wr->IsType(Value::TYPE_DICTIONARY)) {
+        current_wr = new DictionaryValue();
+        web_resource_cache_->Set(wr_counter_str, current_wr);
+    }
+    DictionaryValue* wr_cache_dict =
+      static_cast<DictionaryValue*>(current_wr);
+
+    // Update the resource cache.
+    wr_cache_dict->SetString(L"index", wr_counter_str);
+
+    if (wr_dict->GetString(kWebResourceSnippet, &result_snippet))
+      wr_cache_dict->SetString(kWebResourceSnippet, result_snippet);
+    if (wr_dict->GetString(kWebResourceSource, &result_source))
+      wr_cache_dict->SetString(kWebResourceSource, result_source);
+    if (wr_dict->GetString(kWebResourceURL, &result_url))
+      wr_cache_dict->SetString(kWebResourceURL, result_url);
+    if (wr_dict->GetString(kWebResourceTitle, &result_title))
+      wr_cache_dict->SetString(kWebResourceTitle, result_title);
+    if (wr_dict->GetString(kWebResourceThumb, &result_thumbnail))
+      wr_cache_dict->SetString(kWebResourceThumb, result_thumbnail);
+
+    wr_counter++;
+    wr_iter++;
+  }
+  EndFetch();
+}
+
+void WebResourceService::StartAfterDelay() {
+  int64 delay = kStartResourceFetchDelay;
+  // Check whether we have ever put a value in the web resource cache;
+  // if so, pull it out and see if it's time to update again.
+  if (prefs_->HasPrefPath(prefs::kNTPWebResourceCacheUpdate)) {
+    std::wstring last_update_pref =
+      prefs_->GetString(prefs::kNTPWebResourceCacheUpdate);
+    int64 ms_since_update =
+        (base::Time::Now() - base::Time::FromDoubleT(
+        StringToDouble(WideToASCII(last_update_pref)))).InMilliseconds();
+
+    delay = kStartResourceFetchDelay +
+        (ms_since_update > kCacheUpdateDelay ?
+         0 : kCacheUpdateDelay - ms_since_update);
+  }
+
+  // Start fetch and wait for UpdateResourceCache.
+  DCHECK(delay >= kStartResourceFetchDelay &&
+         delay <= kCacheUpdateDelay);
+  web_resource_fetcher_->StartAfterDelay(static_cast<int>(delay));
+}
+
+void WebResourceService::UpdateResourceCache(const std::string& json_data) {
+  UnpackerClient* client = new UnpackerClient(this, json_data);
+  client->Start();
+
+  // Update resource server and cache update time in preferences.
+  prefs_->SetString(prefs::kNTPWebResourceCacheUpdate,
+      DoubleToWString(base::Time::Now().ToDoubleT()));
+  prefs_->SetString(prefs::kNTPWebResourceServer, web_resource_server_);
+}
+

chrome/browser/web_resource/web_resource_service.h

@@ -0,0 +1,102 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_WEB_RESOURCE_WEB_RESOURCE_SERVICE_H_
+#define CHROME_BROWSER_WEB_RESOURCE_WEB_RESOURCE_SERVICE_H_
+
+#include <string>
+
+#include "chrome/browser/utility_process_host.h"
+#include "chrome/common/pref_service.h"
+#include "chrome/common/web_resource/web_resource_unpacker.h"
+
+class Profile;
+
+class WebResourceService
+    : public UtilityProcessHost::Client {
+ public:
+  WebResourceService(Profile* profile,
+                     MessageLoop* backend_loop);
+  ~WebResourceService();
+
+  // Sleep until cache needs to be updated, but always for at least 5 seconds
+  // so we don't interfere with startup.  Then begin updating resources.
+  void StartAfterDelay();
+
+  // We have successfully pulled data from a resource server; now launch
+  // the process that will parse the JSON, and then update the cache.
+  void UpdateResourceCache(const std::string& json_data);
+
+  // Right now, these values correspond to data pulled from the popgadget
+  // JSON feed.  Once we have decided on the final format for the
+  // web resources servers, these will probably change.
+  static const wchar_t* kWebResourceTitle;
+  static const wchar_t* kWebResourceThumb;
+  static const wchar_t* kWebResourceSource;
+  static const wchar_t* kWebResourceSnippet;
+  static const wchar_t* kWebResourceURL;
+
+  // Default server from which to gather resources.
+  // For now, hard-coded to test JSON data hosted on chromium.org.
+  // Starting 6/22, poptart server will be ready to host data.
+  // Future: more servers and different kinds of data will be served.
+  static const wchar_t* kDefaultResourceServer;
+
+ private:
+  class WebResourceFetcher;
+  friend class WebResourceFetcher;
+
+  class UnpackerClient;
+
+  void Init();
+
+  // Set in_fetch_ to false, clean up temp directories (in the future).
+  void EndFetch();
+
+  // Puts parsed json data in the right places, and writes to prefs file.
+  void OnWebResourceUnpacked(const ListValue& parsed_json);
+
+  // We need to be able to load parsed resource data into preferences file,
+  // and get proper install directory.
+  PrefService* prefs_;
+
+  FilePath web_resource_dir_;
+
+  // Server from which we are currently pulling web resource data.
+  std::wstring web_resource_server_;
+
+  // Whenever we update resource cache, schedule another task.
+  MessageLoop* backend_loop_;
+
+  WebResourceFetcher* web_resource_fetcher_;
+
+  ResourceDispatcherHost* resource_dispatcher_host_;
+
+  // Gets mutable dictionary attached to user's preferences, so that we
+  // can write resource data back to user's pref file.
+  DictionaryValue* web_resource_cache_;
+
+  // True if we are currently mid-fetch.  If we are asked to start a fetch
+  // when we are still fetching resource data, schedule another one in
+  // kCacheUpdateDelay time, and silently exit.
+  bool in_fetch_;
+
+  // Maximum number of cached resources available.
+  static const int kMaxResourceCacheSize = 3;
+
+  // Delay on first fetch so we don't interfere with startup.
+  static const int kStartResourceFetchDelay = 5000;
+
+  // Delay between calls to update the cache (4 hours).
+  static const int kCacheUpdateDelay = 4 * 60 * 60 * 1000;
+
+  // Name of directory inside the profile where we will store resource-related
+  // data (for now, thumbnail images).
+  static const char* kResourceDirectoryName;
+
+  DISALLOW_COPY_AND_ASSIGN(WebResourceService);
+};
+
+#endif  // CHROME_BROWSER_WEB_RESOURCE_WEB_RESOURCE_SERVICE_H_
+

chrome/chrome.gyp

@@ -291,6 +291,8 @@
         'common/net/dns.h',
         'common/net/url_request_intercept_job.cc',
         'common/net/url_request_intercept_job.h',
+        'common/web_resource/web_resource_unpacker.cc',
+        'common/web_resource/web_resource_unpacker.h',
         'common/app_cache/app_cache_context_impl.cc',
         'common/app_cache/app_cache_context_impl.h',
         'common/app_cache/app_cache_dispatcher.cc',
@@ -828,6 +830,8 @@
         'browser/dom_ui/html_dialog_ui.h',
         'browser/dom_ui/new_tab_ui.cc',
         'browser/dom_ui/new_tab_ui.h',
+        'browser/dom_ui/web_resource_handler.cc',
+        'browser/dom_ui/web_resource_handler.h',
         'browser/download/download_exe.cc',
         'browser/download/download_file.cc',
         'browser/download/download_file.h',
@@ -1663,6 +1667,8 @@
         'browser/webdata/web_database.cc',
         'browser/webdata/web_database.h',
         'browser/webdata/web_database_win.cc',
+        'browser/web_resource/web_resource_service.h',
+        'browser/web_resource/web_resource_service.cc',
         'browser/window_sizer.cc',
         'browser/window_sizer.h',
         'browser/window_sizer_mac.mm',

chrome/common/chrome_switches.cc

@@ -486,6 +486,10 @@ const wchar_t kForceFieldTestNameAndValue[]    = L"force-fieldtest";
 // can be a file path, in which case the file will be used as the new tab page.
 const wchar_t kNewNewTabPage[]                 = L"new-new-tab-page";
 
+// Enables the prototype of the backend service for web resources, used in the
+// new new tab page for loading tips and recommendations from a JSON feed.
+const wchar_t kWebResources[]                 = L"enable-web-resources";
+
 // Disables the default browser check. Useful for UI/browser tests where we want
 // to avoid having the default browser info-bar displayed.
 const wchar_t kNoDefaultBrowserCheck[]         = L"no-default-browser-check";

chrome/common/chrome_switches.h

@@ -182,6 +182,8 @@ extern const wchar_t kForceFieldTestNameAndValue[];
 
 extern const wchar_t kNewNewTabPage[];
 
+extern const wchar_t kWebResources[];
+
 extern const wchar_t kEnableBenchmarking[];
 
 extern const wchar_t kNoDefaultBrowserCheck[];

chrome/common/pref_names.cc

@@ -526,4 +526,14 @@ const wchar_t kNTPMostVisitedURLsBlacklist[] = L"ntp.most_visited_blacklist";
 // Page.
 const wchar_t kNTPMostVisitedPinnedURLs[] = L"ntp.pinned_urls";
 
+// Data downloaded from resource pages (JSON, RSS) to be displayed in the
+// recommendations portion of the NTP.
+const wchar_t kNTPWebResourceCache[] = L"ntp.web_resource_cache";
+
+// Last time of update of web_resource_cache.
+const wchar_t kNTPWebResourceCacheUpdate[] = L"ntp.web_resource_update";
+
+// Last server used to fill web_resource_cache.
+const wchar_t kNTPWebResourceServer[] = L"ntp.web_resource_server";
+
 }  // namespace prefs

chrome/common/pref_names.h

@@ -193,6 +193,9 @@ extern const wchar_t kEnableUserScripts[];
 
 extern const wchar_t kNTPMostVisitedURLsBlacklist[];
 extern const wchar_t kNTPMostVisitedPinnedURLs[];
+extern const wchar_t kNTPWebResourceCache[];
+extern const wchar_t kNTPWebResourceCacheUpdate[];
+extern const wchar_t kNTPWebResourceServer[];
 }
 
 #endif  // CHROME_COMMON_PREF_NAMES_H_

chrome/common/render_messages_internal.h

@@ -598,6 +598,12 @@ IPC_BEGIN_MESSAGES(View)
   // Response message to ViewHostMsg_CreateDedicatedWorker.  Sent when the
   // worker has started.
   IPC_MESSAGE_ROUTED0(ViewMsg_DedicatedWorkerCreated)
+
+  // Tell the utility process to parse the given JSON data and verify its
+  // validity.
+  IPC_MESSAGE_CONTROL1(UtilityMsg_UnpackWebResource,
+                       std::string /* JSON data */)
+
 IPC_END_MESSAGES(View)
 
 
@@ -1443,4 +1449,15 @@ IPC_BEGIN_MESSAGES(ViewHost)
   IPC_MESSAGE_CONTROL1(UtilityHostMsg_UnpackExtension_Failed,
                        std::string /* error_message, if any */)
 
+  // Reply when the utility process is done unpacking and parsing JSON data
+  // from a web resource.
+  IPC_MESSAGE_CONTROL1(UtilityHostMsg_UnpackWebResource_Succeeded,
+                       ListValue /* json data */)
+
+  // Reply when the utility process has failed while unpacking and parsing a
+  // web resource.  |error_message| is a user-readable explanation of what
+  // went wrong.
+  IPC_MESSAGE_CONTROL1(UtilityHostMsg_UnpackWebResource_Failed,
+                       std::string /* error_message, if any */)
+
 IPC_END_MESSAGES(ViewHost)

chrome/common/web_resource/web_resource_unpacker.cc

@@ -0,0 +1,38 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/common/web_resource/web_resource_unpacker.h"
+
+#include "base/json_reader.h"
+#include "base/values.h"
+
+const char* WebResourceUnpacker::kInvalidDataTypeError =
+    "Data from web resource server is missing or not valid JSON.";
+
+const char* WebResourceUnpacker::kUnexpectedJSONFormatError =
+    "Data from web resource server does not have expected format.";
+
+// TODO(mrc): Right now, this reads JSON data from the experimental popgadget
+// server.  Change so the format is based on a template, once we have
+// decided on final server format.
+bool WebResourceUnpacker::Run() {
+  scoped_ptr<Value> value;
+  if (!resource_data_.empty()) {
+    value.reset(JSONReader::Read(resource_data_, false));
+    if (!value.get()) {
+      // Page information not properly read, or corrupted.
+      error_message_ = kInvalidDataTypeError;
+      return false;
+    }
+    if (!value->IsType(Value::TYPE_LIST)) {
+      error_message_ = kUnexpectedJSONFormatError;
+      return false;
+    }
+    parsed_json_.reset(static_cast<ListValue*>(value.release()));
+    return true;
+  }
+  error_message_ = kInvalidDataTypeError;
+  return false;
+}
+

chrome/common/web_resource/web_resource_unpacker.h

@@ -0,0 +1,57 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This class is called by the WebResourceService in a sandboxed process
+// to unpack data retrieved from a web resource feed.  Right now, it
+// takes a string of data in JSON format, parses it, and hands it back
+// to the WebResourceService as a list of items.  In the future
+// it will be set up to unpack and verify image data in addition to
+// just parsing a JSON feed.
+
+#ifndef CHROME_COMMON_WEB_RESOURCE_UNPACKER_H_
+#define CHROME_COMMON_WEB_RESOURCE_UNPACKER_H_
+
+#include <string>
+
+#include "base/file_path.h"
+#include "base/scoped_ptr.h"
+
+class ListValue;
+
+class WebResourceUnpacker {
+ public:
+  static const char* kInvalidDataTypeError;
+  static const char* kUnexpectedJSONFormatError;
+
+  explicit WebResourceUnpacker(const std::string &resource_data)
+    : resource_data_(resource_data) {}
+
+  // This does the actual parsing.  In case of an error, error_message_
+  // is set to an appropriate value.
+  bool Run();
+
+  // Returns the last error message set by Run().
+  const std::string& error_message() { return error_message_; }
+
+  // Gets data which has been parsed by Run().
+  ListValue* parsed_json() {
+    return parsed_json_.get();
+  }
+
+ private:
+  // Holds the string which is to be parsed.
+  std::string resource_data_;
+
+  // Holds the result of JSON parsing of resource_data_.
+  scoped_ptr<ListValue> parsed_json_;
+
+  // Holds the last error message produced by Run().
+  std::string error_message_;
+
+  DISALLOW_COPY_AND_ASSIGN(WebResourceUnpacker);
+};
+
+#endif  // CHROME_COMMON_WEB_RESOURCE_UNPACKER_H_
+
+

chrome/test/testing_profile.h

@@ -205,6 +205,8 @@ class TestingProfile : public Profile {
   }
   virtual void InitExtensions() {
   }
+  virtual void InitWebResources() {
+  }
 
   // Schedules a task on the history backend and runs a nested loop until the
   // task is processed.  This has the effect of blocking the caller until the

chrome/utility/utility_thread.cc

@@ -6,6 +6,7 @@
 
 #include "base/file_util.h"
 #include "base/values.h"
+#include "chrome/common/web_resource/web_resource_unpacker.h"
 #include "chrome/common/child_process.h"
 #include "chrome/common/extensions/extension_unpacker.h"
 #include "chrome/common/render_messages.h"
@@ -29,6 +30,7 @@ void UtilityThread::CleanUp() {
 void UtilityThread::OnControlMessageReceived(const IPC::Message& msg) {
   IPC_BEGIN_MESSAGE_MAP(UtilityThread, msg)
     IPC_MESSAGE_HANDLER(UtilityMsg_UnpackExtension, OnUnpackExtension)
+    IPC_MESSAGE_HANDLER(UtilityMsg_UnpackWebResource, OnUnpackWebResource)
   IPC_END_MESSAGE_MAP()
 }
 
@@ -43,3 +45,20 @@ void UtilityThread::OnUnpackExtension(const FilePath& extension_path) {
 
   ChildProcess::current()->ReleaseProcess();
 }
+
+void UtilityThread::OnUnpackWebResource(const std::string& resource_data) {
+  // Parse json data.
+  // TODO(mrc): Add the possibility of a template that controls parsing, and
+  // the ability to download and verify images.
+  WebResourceUnpacker unpacker(resource_data);
+  if (unpacker.Run()) {
+    Send(new UtilityHostMsg_UnpackWebResource_Succeeded(
+        *unpacker.parsed_json()));
+  } else {
+    Send(new UtilityHostMsg_UnpackWebResource_Failed(
+        unpacker.error_message()));
+  }
+
+  ChildProcess::current()->ReleaseProcess();
+}
+

chrome/utility/utility_thread.h

@@ -5,6 +5,8 @@
 #ifndef CHROME_UTILITY_UTILITY_THREAD_H_
 #define CHROME_UTILITY_UTILITY_THREAD_H_
 
+#include <string>
+
 #include "base/thread.h"
 #include "chrome/common/child_thread.h"
 
@@ -26,6 +28,9 @@ class UtilityThread : public ChildThread {
   virtual void OnControlMessageReceived(const IPC::Message& msg);
   void OnUnpackExtension(const FilePath& extension_path);
 
+  // IPC messages for web resource service.
+  void OnUnpackWebResource(const std::string& resource_data);
+
   // Called by the thread base class
   virtual void Init();
   virtual void CleanUp();