1
1
2
2
require ( "whatwg-fetch" ) ;
3
3
var URI = require ( "../utils/uri.js" ) . URI ;
4
- var Options = require ( "../utils/options.js" ) ;
5
-
6
- var MAX_CONCURRENT_REQUESTS = 100 ; // Maximum number of requests awaiting a response
7
4
8
5
var c_requestHooks = [ ] ; // Request hooks called for each outgoing request
9
6
var c_formatHandlers = [ ] ; // All registered FormatHandler plugins
@@ -22,21 +19,26 @@ Resource.fetch = function(uriString, opt) {
22
19
opt = initOptions ( opt ) ;
23
20
var uri = new URI ( uriString ) ;
24
21
22
+ for ( var i = 0 ; i < c_requestHooks . length ; i ++ ) {
23
+ c_requestHooks [ i ] ( uri , opt ) ;
24
+ }
25
+ if ( opt . abort ) {
26
+ return Promise . reject ( new RequestAbortedException ( uri ) ) ;
27
+ }
28
+
29
+ return doFetch ( uri , opt ) ;
30
+ } ;
31
+
32
+ function doFetch ( uri , opt ) {
25
33
return new Promise ( function ( resolve , reject ) {
26
- for ( var i = 0 ; i < c_requestHooks . length ; i ++ ) {
27
- c_requestHooks [ i ] ( uri , opt ) ;
28
- }
29
- if ( opt . abort ) {
30
- throw new RequestAbortedException ( uri ) ;
31
- }
32
34
scheduleRequest ( {
33
35
opt : opt ,
34
36
uri : uri ,
35
37
resolve : resolve ,
36
38
reject : reject
37
39
} ) ;
38
40
} ) ;
39
- } ;
41
+ }
40
42
41
43
var popRequestQueue = function ( ) {
42
44
var request = c_requestQueue . shift ( ) ;
@@ -54,21 +56,38 @@ var popRequestQueue = function() {
54
56
} ;
55
57
56
58
Resource . getDocument = function ( urlString , opt ) {
57
- return Resource . fetch ( urlString , opt )
59
+ opt = initOptions ( opt ) ;
60
+ var uri = new URI ( urlString ) ;
61
+
62
+ for ( var i = 0 ; i < c_requestHooks . length ; i ++ ) {
63
+ c_requestHooks [ i ] ( uri , opt ) ;
64
+ }
65
+ if ( opt . abort ) {
66
+ Promise . reject ( new RequestAbortedException ( uri ) ) ;
67
+ }
68
+
69
+ var cache ;
70
+ if ( opt . allowCaching ) {
71
+ if ( cache = c_cachedDocuments . get ( urlString ) ) {
72
+ // We can skip the fetch phase and piggy back on the result of the previous fetch, this avoids unnecessary Requests
73
+ return cache . pending ? cache . pending : Promise . resolve ( cache . document ) ;
74
+ }
75
+
76
+ // There is no pending or complete Request for this url so lets create a cache entry and start one
77
+ cache = { fragments : [ ] } ;
78
+ c_cachedDocuments . set ( urlString , cache ) ; // Resource.parseResponse expects this entry to exist already
79
+ }
80
+
81
+ var documentPromise = doFetch ( urlString , opt )
58
82
. then ( function ( response ) {
59
83
if ( ! response . ok ) {
60
84
throw new RequestFailedException ( response ) ;
61
85
}
62
86
response . originalURL = urlString ;
63
- var cache ;
64
- if ( cache = c_cachedDocuments . get ( urlString ) ) {
65
- return cache . pending ? cache . pending : cache . document ;
66
- } else {
67
- cache = { fragments : [ ] } ;
68
- c_cachedDocuments . set ( urlString , cache ) ; // Resource.parseResponse expects this entry to exist already
69
- cache . pending = Resource . parseResponse ( response ) ;
70
- return cache . pending ;
71
- }
87
+ var cache = { fragments : [ ] } ;
88
+ c_cachedDocuments . set ( urlString , cache ) ; // Resource.parseResponse expects this entry to exist already
89
+ cache . pending = Resource . parseResponse ( response ) ;
90
+ return cache . pending ;
72
91
} ) . then ( function ( doc ) {
73
92
doc . _documentURL = urlString ;
74
93
var cache = c_cachedDocuments . get ( urlString ) ;
@@ -79,6 +98,12 @@ Resource.getDocument = function(urlString, opt) {
79
98
c_cachedDocuments . has ( urlString ) && delete c_cachedDocuments . get ( urlString ) . pending ;
80
99
throw exception ;
81
100
} ) ;
101
+
102
+ if ( cache ) {
103
+ cache . pending = documentPromise ;
104
+ }
105
+
106
+ return documentPromise ;
82
107
} ;
83
108
84
109
Resource . parseResponse = function ( response ) {
@@ -144,7 +169,7 @@ var prioritySort = function(a, b) {
144
169
145
170
var tickWorkWindow = function ( ) {
146
171
// Both of these loops trigger asynchronous work through Promises so working through all queue items shouldn't block the thread for too long
147
- while ( c_requestQueue . length > 0 && c_openRequests < MAX_CONCURRENT_REQUESTS ) {
172
+ while ( c_requestQueue . length > 0 ) {
148
173
popRequestQueue ( ) ;
149
174
}
150
175
@@ -159,6 +184,7 @@ var initOptions = function(opt) {
159
184
opt . headers = opt . headers || { } ;
160
185
opt . priority = opt . priority || 0 ;
161
186
opt . abort = opt . abort || false ;
187
+ opt . allowCaching = opt . allowCaching !== undefined ? opt . allowCaching : true ;
162
188
return opt ;
163
189
} ;
164
190
0 commit comments