up

Author: geraldb

gemverse/.gitignore

@@ -0,0 +1,41 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+
+####
+##  add cache for gems api requests
+cache/
+

gemverse/NOTES.md

@@ -3,8 +3,15 @@
 ## Gem Gems In Ruby
 
 
+###  Gem Stats & More
+
+**whatthegem** && **any_good**  see
+- <https://github.com/zverok/whatthegem>
+- <https://github.com/zverok/any_good>
+
 
 
 ### Gem API Wrappers / Calls
 
-**Gems** see <https://github.com/rubygems/gems>
+**gems** see
+- <https://github.com/rubygems/gems>

gemverse/gems/README.md

@@ -1,7 +1,7 @@
 name, version, created, downloads
-ethlite-contracts, 0.1.4, 2023-01-16, 108
-ethlite-contracts, 0.1.3, 2023-01-15, 108
-ethlite-contracts, 0.1.2, 2023-01-15, 75
-ethlite-contracts, 0.1.1, 2023-01-13, 126
-ethlite-contracts, 0.1.0, 2023-01-12, 380
+ethlite-contracts, 0.1.4, 2023-01-16, 111
+ethlite-contracts, 0.1.3, 2023-01-15, 109
+ethlite-contracts, 0.1.2, 2023-01-15, 76
+ethlite-contracts, 0.1.1, 2023-01-13, 127
+ethlite-contracts, 0.1.0, 2023-01-12, 420
 ethlite-contracts, 0.0.1, 2023-01-09, 160

gemverse/gems/ethlite-contracts/versions.csv

@@ -1,15 +1,15 @@
 name, version, created, downloads
-eventdb, 1.1.2, 2023-01-18, 40
-eventdb, 1.1.1, 2019-12-08, 2036
-eventdb, 1.1.0, 2019-12-08, 1440
-eventdb, 1.0.0, 2019-12-06, 1475
-eventdb, 0.7.1, 2019-12-05, 1470
-eventdb, 0.7.0, 2015-07-12, 2664
-eventdb, 0.6.1, 2015-07-10, 2061
-eventdb, 0.6.0, 2015-06-24, 2075
-eventdb, 0.5.0, 2015-06-23, 2068
-eventdb, 0.4.0, 2015-06-23, 2039
-eventdb, 0.3.0, 2015-06-22, 2065
-eventdb, 0.2.0, 2015-06-21, 2059
-eventdb, 0.1.0, 2015-06-21, 2034
-eventdb, 0.0.1, 2015-06-21, 2036
+eventdb, 1.1.2, 2023-01-18, 99
+eventdb, 1.1.1, 2019-12-08, 2037
+eventdb, 1.1.0, 2019-12-08, 1441
+eventdb, 1.0.0, 2019-12-06, 1476
+eventdb, 0.7.1, 2019-12-05, 1471
+eventdb, 0.7.0, 2015-07-12, 2665
+eventdb, 0.6.1, 2015-07-10, 2062
+eventdb, 0.6.0, 2015-06-24, 2076
+eventdb, 0.5.0, 2015-06-23, 2069
+eventdb, 0.4.0, 2015-06-23, 2040
+eventdb, 0.3.0, 2015-06-22, 2066
+eventdb, 0.2.0, 2015-06-21, 2060
+eventdb, 0.1.0, 2015-06-21, 2035
+eventdb, 0.0.1, 2015-06-21, 2037

gemverse/gems/eventdb/versions.csv

@@ -1,5 +1,5 @@
 name, version, created, downloads
-markdown, 1.2.0, 2014-12-06, 72682
+markdown, 1.2.0, 2014-12-06, 72717
 markdown, 1.1.1, 2013-05-25, 50614
 markdown, 1.1.0, 2013-05-24, 3038
 markdown, 1.0.0, 2012-06-23, 81896

gemverse/gems/markdown/versions.csv

@@ -1,5 +1,5 @@
 name, version, created, downloads
-whatson, 2023.0.0, 2023-01-18, 37
+whatson, 2023.0.0, 2023-01-18, 96
 whatson, 1.1.0, 2019-12-08, 1890
 whatson, 1.0.0, 2019-12-06, 1337
 whatson, 0.1.0, 2015-06-23, 2720

gemverse/gems/whatson/versions.csv

@@ -1,48 +1,95 @@
 require 'cocos'
 
 
+## move to cocos - upstream - why? why not?
+def write_csv( path, recs )
+  buf = String.new
+  buf << recs[0].join( ', ' )
+  buf << "\n"
+  recs[1..-1].each do |values|
+    buf << values.join( ', ' )
+    buf << "\n"
+  end
+  write_text( path, buf )
+end
 
-module Gems
 
-def self.gems_by( handle )
-  src = "#{BASE}/owners/#{handle}/gems.json"
-  call( src )
-end
 
-def self.versions( name )
 
-  ## note: will NOT include yanked versions
-  ##   check if there's a query parameter ???
 
 
-  src = "#{BASE}/versions/#{name}.json"
-  call( src )
-end
+require_relative 'gemverse/api'
+require_relative 'gemverse/gems'
+require_relative 'gemverse/cache'
 
+require_relative 'gemverse/timeline'  ## timeline report
 
 
-BASE = 'https://rubygems.org/api/v1'
 
+module Gems
+
+###
+##  "high-level" finders
+
+def self.find_by( owner: )  ## todo/check: use
 
-def self.call( src )   ## get response as (parsed) json (hash table)
+  rows = API.gems_by( owner )
+  # pp data
+  puts "  #{rows.size} record(s) founds"
 
-  headers = {
-    # 'User-Agent' => "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
-    'User-Agent' => "ruby v#{RUBY_VERSION}",
-   }
+  ## write "raw respone" to cache for debugging
+  write_json( "./cache/gems_#{owner}.json", rows )
 
-  response = Webclient.get( src, headers: headers )
 
-  if response.status.ok?
-    puts "#{response.status.code} #{response.status.message} -  content_type: #{response.content_type}, content_length: #{response.content_length}"
 
-    response.json
-  else
-    puts "!! HTTP ERROR:"
-    puts "#{response.status.code} #{response.status.message}"
-    exit 1
+  recs = []
+  rows.each do |row|
+     recs <<  Gem.create( row )
   end
+
+  ## sort by
+  ##  1) (latest) version created and
+  ##  2) name
+   recs = recs.sort do |l,r|
+     res =  r.version_created <=> l.version_created
+     res =  l.name            <=> r.name       if res == 0
+     res
+   end
+
+  GemDataset.new( recs )
+end
+
+
+
+def self.read_csv( path )
+  ## note: requires Kernel::  otherwise stackoverflow endlessly calling read_csv
+  rows = Kernel::read_csv( path )
+  puts "  #{rows.size} record(s) founds"
+
+  recs = []
+  rows.each do |row|
+     kwargs = {
+      version: row['version'].empty? ? nil : row['version'],
+      version_downloads: row['version_downloads'].empty? ? nil : row['version_downloads'].to_i,
+      version_created:  row['version_created'].empty? ? nil : Date.strptime( row['version_created'], '%Y-%m-%d' ),
+      homepage:  row['homepage'].empty? ? nil : row['homepage'],
+      runtime:   row['dependencies'].empty? ? [] : row['dependencies'].split('|').map {|dep| dep.strip },
+     }
+     recs <<  Gem.new( name: row['name'], **kwargs )
+  end
+
+  ## sort by
+  ##  1) (latest) version created and
+  ##  2) name
+   recs = recs.sort do |l,r|
+     res =  r.version_created <=> l.version_created
+     res =  l.name            <=> r.name     if res == 0
+     res
+   end
+
+  GemDataset.new( recs )
 end
 
 
 end  # module Gems
+

gemverse/lib/gemverse.rb

@@ -0,0 +1,42 @@
+
+module Gems
+module API
+
+BASE = 'https://rubygems.org/api/v1'
+
+def self.gems_by( owner )
+  src = "#{BASE}/owners/#{owner}/gems.json"
+  call( src )
+end
+
+
+def self.versions( name )
+  ## note: will NOT include yanked versions
+  ##   check if there's a query parameter ???
+  src = "#{BASE}/versions/#{name}.json"
+  call( src )
+end
+
+
+def self.call( src )   ## get response as (parsed) json (hash table)
+
+  headers = {
+    # 'User-Agent' => "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
+    'User-Agent' => "ruby v#{RUBY_VERSION}",
+   }
+
+  response = Webclient.get( src, headers: headers )
+
+  if response.status.ok?
+    puts "#{response.status.code} #{response.status.message} -  content_type: #{response.content_type}, content_length: #{response.content_length}"
+
+    response.json
+  else
+    puts "!! HTTP ERROR:"
+    puts "#{response.status.code} #{response.status.message}"
+    exit 1
+  end
+end
+
+end  # module API
+end  # module Gems

gemverse/lib/gemverse/api.rb

@@ -0,0 +1,100 @@
+
+
+module Gems
+class Cache
+def initialize( basedir='./gems' )
+  @basedir = basedir
+end
+
+
+def update_versions( gems: [] )
+
+  delay_in_s = 0.5
+
+  gems.each_with_index do |gem,i|
+
+    name = if gem.is_a?( String )
+                gem
+           elsif gem.is_a?( Hash )
+                gem['name']
+           else  ## assume our own Gem struct/class for now
+                gem.name
+           end
+
+    puts "==>  #{i+1}/#{gems.size} #{name}..."
+
+    ## update versions info
+    puts "  sleeping #{delay_in_s} second(s)"
+    sleep( delay_in_s )
+
+    data = Gems::API.versions( name )
+    puts "   #{data.size} version record(s) found"
+
+    headers = ['name',
+               'version',
+               'created',
+               'downloads']
+    recs = []
+    data.each do |h|
+      ## only use year/month/day for now now hours etc.
+      created = Date.strptime( h['created_at'], '%Y-%m-%d' )
+
+       rec = [
+         name,
+         h['number'],
+         created.strftime( '%Y-%m-%d' ),
+         h['downloads_count'].to_s,
+       ]
+       recs << rec
+    end
+    write_csv( "#{@basedir}/#{name}/versions.csv", [headers]+recs )
+  end
+end
+
+
+
+def read_versions( gems: [] )
+  ## read in and merge all version records
+versions = []
+
+ ## if no gems passed in - get all versions.csv datasets in basedir
+ if gems.empty?
+   paths = Dir.glob( "#{@basedir}/*/versions.csv" )
+   gems = paths.map { |path| File.basename(File.dirname(path)) }
+ end
+
+  gems.each_with_index do |gem,i|
+
+    name = if gem.is_a?( String )
+                gem
+           elsif gem.is_a?( Hash )
+                gem['name']
+           else  ## assume our own Gem struct/class for now
+                gem.name
+           end
+
+    path = "#{@basedir}/#{name}/versions.csv"
+    puts "==> #{i+1}/#{gems.size} reading #{name}..."
+    recs = read_csv( path )
+    recs.reverse.each_with_index do |rec,n|
+        more = { 'count' => (n+1).to_s }   ## auto-add version count(er) e.g. 1,2,3,...
+        versions <<  rec.merge( more )
+    end
+    puts "   #{recs.size} record(s)"
+  end
+
+  ## sort by version created and name
+  versions = versions.sort do |l,r|
+    res =  r['created'] <=> l['created']
+    res =  l['name']    <=> r['name']     if res == 0
+    res =  r['version'] <=> l['version']  if res == 0
+    res
+  end
+
+  versions
+end
+
+
+
+end   ## class Cache
+end   ## module Gems