#!/usr/pkg/bin/ruby26
# $Id: scrape-MDAPI-2-geogroups.rb 6954 2018-12-24 21:55:47Z flaterco $
# API documented at: https://tidesandcurrents.noaa.gov/mdapi/latest/
# If it should go away, there is another one at https://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/
# The geogroups query is undocumented.

require 'json'

# Get the complete list of geogroups with parent pointers.
unless File.exist?("geogroups-tides.json")
  system "wget --progress=dot -O geogroups-tides.json https://tidesandcurrents.noaa.gov/mdapi/latest/webapi/geogroups.json?type=ETIDES"
end

# Make list of URLs to get the stations in each geogroup.
unless File.exist?("geogroups-tides-urls.txt")
  geogroups = JSON.parse(File.read("geogroups-tides.json"))
  File.open("geogroups-tides-urls.txt", "w") do |outf|
    geogroups["geoGroupList"].each {|g|
      outf.print "https://tidesandcurrents.noaa.gov/mdapi/latest/webapi/geogroups/", g["geoGroupId"], "/children.json\n"
    }
  end
end

# Not sure what the seq field is for.

# The records all contain pointers to the parent geogroup, so it's not
# necessary to save separate files for each get.
unless File.exist?("stations-by-group-tides.json")
  system "wget -O stations-by-group-tides.json --wait=4 --random-wait --no-clobber --input-file=geogroups-tides-urls.txt --append-output=scrape_log_geogroups_tides.txt"
end


# Commented-out dead code follows.
#
# 2017:  Geogroup children query (getting stations) does not work for
#        currents.  It retrieves only the group records.
# 2018:  Bag it:  none of this will be needed or used until there are new
#        current stations.
=begin

unless File.exist?("geogroups-currents.json")
  system "wget --progress=dot -O geogroups-currents.json https://tidesandcurrents.noaa.gov/mdapi/latest/webapi/geogroups.json?type=CURRENTS"
end

unless File.exist?("geogroups-currents-urls.txt")
  geogroups = JSON.parse(File.read("geogroups-currents.json"))
  File.open("geogroups-currents-urls.txt", "w") do |outf|
    geogroups["geoGroupList"].each {|g|
      outf.print "https://tidesandcurrents.noaa.gov/mdapi/latest/webapi/geogroups/", g["geoGroupId"], "/children.json\n"
    }
  end
end

# This retrieves nothing but the group records.
unless File.exist?("stations-by-group-currents.json")
  system "wget -O stations-by-group-currents.json --wait=4 --random-wait --no-clobber --input-file=geogroups-currents-urls.txt --append-output=scrape_log_geogroups_currents.txt"
end

=end
