Project

General

Profile

Paste
Download (15.5 KB) Statistics
| Branch: | Revision:

root / other-scripts / one-use / migrate-issues-to-redmine / goto_redmine.py @ 3a2b15dd

1
#!/usr/bin/env python3
2

    
3
"""This script can migrate issues from drupal to redmine.
4

5
It is design be launched with the -i option: most variables you may need for
6
debugging are easily accessible from interpreter (like HTTP status codes).
7
We make some assertion based on HTTP status code. Here are the codet you may
8
wish to know:
9
- 200: OK
10
- 201: Created
11
- 404: Not found
12
- 403: Forbidden
13
- 500: Internal error
14
Redmine gives you more intel in its response. Read them!
15
Pandoc is required to convert html syntax to textile syntax
16
"""
17

    
18
import requests
19
import json
20
import sys
21
import datetime
22
import os #we need system() to call pandoc
23
import re
24

    
25
import constantes as cst
26

    
27
######## Global variables
28
REGEXP_FIND_IMG = re.compile('!/.*!')
29
REGEXP_NAME_IMG = re.compile('!.*/(.*)!')
30

    
31
######## Common functions
32

    
33
def handle_image(txt):
34
    "Images are not posted automatically. There are only few of them.\
35
    We just format the text with the correct textile syntax and when we post them,\
36
    we will add comment and node id into a file. They should be attached to the\
37
    correct issue afterwards"
38
    has_image = False
39
    # In textile, images are between !
40
    images = REGEXP_FIND_IMG.findall(txt)
41
    if images:
42
        has_image = True
43
        for image in images:
44
            img_name = REGEXP_NAME_IMG.sub(r'!\1!', image)
45
            txt = txt.replace(image, img_name)
46
    return txt, has_image
47

    
48
def html2textile(txt):
49
    "Convert a txt from html to textile using pandoc"
50
    # We remove line breaks and tabs, otherwise the conversion doesn't work properly
51
    txt.replace('\n', '')
52
    txt.replace('\t', '')
53
    # pandoc can only manipulates files
54
    with open('tmp.html', 'w') as f:
55
        f.write(txt)
56
    os.system('pandoc -f html tmp.html -t textile -o tmp.textile')
57
    with open('tmp.textile', 'r') as f:
58
        txt = f.read()
59
    # Cleaning temporary files
60
    os.remove('tmp.html')
61
    os.remove('tmp.textile')
62
    return handle_image(txt)
63

    
64
def egalise(string, length):
65
    "Make the length of string equals to length if shorter"
66
    if len(string) < length:
67
        return ' '*(length - len(string)) + string
68
    else:
69
        return string
70

    
71
def percentage(integer):
72
    "Converts integer into a string used to indicate the percentage of completion\
73
    of a command"
74
    string = str(integer)
75
    string = egalise(string, 3)
76
    string = 'Completion: ' + string + '%'
77
    return string + '\b'*len(string)
78

    
79
def print_progress(str):
80
    sys.stdout.write(str)
81
    sys.stdout.flush()
82

    
83
def format_date(timestamp):
84
    str_timestamp = float(timestamp)
85
    date = datetime.datetime.fromtimestamp(str_timestamp)
86
    return date.strftime('%Y-%m-%d %H:%M:%S')
87

    
88

    
89

    
90
######## Definition of classes
91

    
92

    
93
class Comment:
94
    """Represents a drupal comment
95
    """
96

    
97
    def __init__(self, cid, author, post_date, content, has_img):
98
        self._cid = cid # comment id in drupal
99
        self._author = author
100
        self._post_date = post_date
101
        self._content = content
102
        # json representation, to be posted in redmine
103
        self._update = {'issue': {'notes': self._content }}
104
        self._update_json = json.dumps(self._update)
105
        self._resp = None #will be used to store the put response
106
        self._has_img = has_img
107

    
108
    def post(self, url, headers, iid, post_nb):
109
        "Post the comment to url with headers (required for authentication)"
110
        self._resp = requests.put(url, headers=headers, data=self._update_json)
111
        assert self._resp.status_code == 200
112

    
113
        # We write iid,author_id,created_on in comments.csv
114
        with open('comments.csv', 'a', encoding='utf8') as comments_csv:
115
            comments_csv.write('{},{},{}\n'.\
116
                        format(iid, cst.USER_ID[self._author], self._post_date))
117
        with open('fix_url_comments.csv', 'a', encoding='utf8') as fix_url_csv:
118
            fix_url_csv.write('{},{},{}\n'.format(self._cid, iid, post_nb))
119

    
120
    @property
121
    def post_date(self):
122
        return self._post_date
123

    
124
    @property
125
    def resp(self):
126
        return self._resp
127

    
128
    @property
129
    def cid(self):
130
        return self._cid
131

    
132
    @property
133
    def has_img(self):
134
        return self._has_img
135

    
136

    
137

    
138
class Updates:
139
    """Represents all the comments of a task
140
    """
141

    
142
    def __init__(self, comments):
143
        self._comments = comments
144

    
145
    def sort(self):
146
        "Sort all the updates by date of creation"
147
        sorted_date = False
148
        while not sorted_date:
149
            sorted_date = True
150
            i = 0
151
            while i < len(self._comments) - 1:
152
                if self._comments[i].post_date > self._comments[i + 1].post_date:
153
                    self._comments[i], self._comments[i + 1] = self._comments[i + 1],\
154
                                                               self._comments[i]
155
                    sorted_date = False
156
                i += 1
157

    
158
    def __getitem__(self, index):
159
        return self._comments[index]
160

    
161
    def __len__(self):
162
        return len(self._comments)
163

    
164
    def __iter__(self):
165
        self.__i = -1
166
        return self
167

    
168
    def __next__(self):
169
        self.__i += 1
170
        if self.__i >= len(self._comments) or len(self._comments) == 0:
171
            raise StopIteration
172
        return self._comments[self.__i]
173

    
174

    
175

    
176
class Issue:
177
    """Represents a drupal issue
178
    """
179

    
180
    def __init__(self, nid, comments):
181
        self._nid = nid #node id
182
        self._iid = None #issue id, unknown until creation
183
        self._resp = None #will be used to store the response of requests.post
184
        self._comments = Updates(comments)
185
        self._comments.sort()
186
        self._issue = self.give_redmine_issue(nid) #the actual content, it's a dict
187

    
188
    def give_redmine_status_id(self, node):
189
        "Translate the drupal status field to an integer representing the\
190
        redmine status id"
191
        drupal_status = ''
192
        for elt in node['field_avancement']:
193
            if "Terminée" in elt:
194
                drupal_status = 'Fermée'
195
                break
196
            elif "Fermée" in elt:
197
                drupal_status = 'Rejetée'
198
                break
199
            elif "pause" in elt:
200
                drupal_status = 'En pause'
201
                del elt
202
                break
203
        if not drupal_status:
204
            drupal_status = 'En cours'
205
        return cst.STATUS[drupal_status]
206

    
207
    def give_redmine_issue(self, nid):
208
        "Uses the nid to find the node and converts its content to something\
209
        redmine can understand. Read examples for more intels"
210
        node_json = requests.get(cst.BASE_URL + '/node/{}.json'.format(nid)).text
211
        node = json.loads(node_json)
212
        issue = dict()
213
        issue['project_id'] = cst.PROJECT_ID
214
        issue['tracker_id'] = cst.TRACKER_ID
215
        issue['subject'] = node['title']
216
        issue['description'], self._has_img = html2textile(node['body']['value'])
217
        # We get the name of the node
218
        self._name = re.findall(cst.REGEXP_NAME, node['url'])[0]
219
        # field_prioritaecute can be empty. We then assume it is normal
220
        if node['field_prioritaecute']:
221
            issue['priority_id'] = cst.PRIORITY[node['field_prioritaecute']]
222
        else:
223
            issue['priority_id'] = cst.PRIORITY['3 - Moyenne']
224
        # field_avancement can be empty. We then assume it is to be started
225
        if node['field_avancement']:
226
            issue['done_ratio'] = cst.DONE_RATIO[node['field_avancement'][0]]
227
        else:
228
            issue['done_ratio'] = cst.DONE_RATIO['À commencer']
229
        # Status id = open, fix, closed…
230
        issue['status_id'] = self.give_redmine_status_id(node)
231
        issue['fixed_version_id'] = cst.DRUPAL_VERSION[node['taxonomy_vocabulary_8']['id']]
232
        issue['created'] = format_date(node['created'])
233
        issue['author_id'] = node['author']['id']
234
        # Do we have attached files?
235
        if node['field_fichier']:
236
            self._has_files = True
237
        else:
238
            self._has_files = False
239
        return issue
240

    
241
    def post(self, url, headers):
242
        "Post the comment to url with headers (required for authentication)"
243
        issue = {'issue': self._issue}
244
        data = json.dumps(issue)
245

    
246
        self._resp = requests.post(url, headers=headers, data=data)
247

    
248
        assert self._resp.status_code == 201
249

    
250
        resp_json = json.loads(self._resp.text)
251
        self._iid = resp_json['issue']['id']
252

    
253
        # We write iid,author_id,created_on in issues.csv
254
        with open('issues.csv', 'a', encoding='utf8') as issues_csv:
255
            author_id = self._issue['author_id']
256
            redmine_author_id = cst.USER_ID[author_id]
257
            created_on = self._issue['created']
258
            issues_csv.write('{},{},{}\n'.\
259
                             format(self._iid, redmine_author_id, created_on))
260
        with open('fix_url_issues.csv', 'a') as fix_url:
261
            fix_url.write('{},{},{}\n'.format(self._nid, self._name, self._iid))
262

    
263
        # We post comments
264
        nb_comments = len(self._comments)
265
        i = 0
266
        for comment in self._comments:
267
            i += 1
268
            put_url =  cst.URL_ISSUES + '/{}.json'.format(self._iid)
269
            comment.post(put_url, headers, self._iid, i)
270
            print_progress(percentage(i//nb_comments*100))
271

    
272
        # We take care of images and files
273
        self.handle_image()
274
        self.handle_files()
275

    
276
    def handle_files(self):
277
        if self._has_files:
278
            with open('has_files.txt', 'a', encoding='utf8') as has_files_file:
279
                has_files_file.write('{}\n'.format(self._nid))
280

    
281
    def handle_image(self):
282
        for comment in self._comments:
283
            self._has_img = comment.has_img or self._has_img
284
        if self._has_img:
285
            with open('has_img.txt', 'a', encoding='utf8') as has_img_file:
286
                has_img_file.write('{}\n'.format(self._nid))
287

    
288
    @property
289
    def resp(self):
290
        return self._resp
291

    
292
    @property
293
    def comments_resp(self):
294
        resps = dict()
295
        for comment in self._comments:
296
            resps[comment.cid] = comment.resp
297
        return resps
298

    
299

    
300

    
301
class Laundry:
302
    """Contains all issues and has methods to perform the migration
303

304
    Iteration of this object traverses all issues.
305
    You can access any issue with the container notation
306
    """
307

    
308
    def __init__(self, url, test=True):
309
        self._issues = self.give_issues(url, test)
310

    
311
    def give_issues(self, url, test):
312
        "Returns a list of all issues"
313
        issues = []
314
        r = requests.get(url)
315

    
316
        assert r.status_code == 200
317

    
318
        # 1st element is 'Nid', and last is ''
319
        nids = r.text.split('\r\n')[1:-1]
320
        # for test, we only use 3 issues (faster)
321
        if test:
322
            nids = nids[:3]
323
        self.nb_task = len(nids)
324
        i = 0
325
        for nid in nids:
326
            i += 1
327
            comments = self.give_comments(nid)
328
            print('Fetching issue no {} on {}'.format(i, self.nb_task))
329
            issues.append(Issue(nid, comments))
330
        return issues
331

    
332
    def give_comments(self, nid):
333
        "Returns the list of all comments of a node"
334
        cids, comments_json = self.give_comments_json(nid)
335
        comments = []
336
        i = 0
337
        nb_comments = len(comments_json)
338
        for comment_json in comments_json:
339
            author = comment_json['name']
340
            post_date = format_date(comment_json['created'])
341
            content, has_img = html2textile(comment_json['comment_body']['value'])
342
            comments.append(Comment(cids[comments_json.index(comment_json)], author, post_date, content, has_img))
343
            i += 1
344
            print_progress(percentage(i//nb_comments*100))
345
        return comments
346

    
347
    def give_comments_json(self, nid):
348
        "Get the raw json version of the drupal comment"
349
        cids = self.give_comments_ids(nid)
350
        comments = list()
351
        for cid in cids:
352
            comment = requests.get(cst.BASE_URL + '/comment/' + cid + '.json')
353
            comments.append(json.loads(comment.text))
354
        return cids, comments
355

    
356
    def give_comments_ids(self, nid):
357
        "Get the cid (comnments id) for a node"
358
        headers = cst.Headers_GET
359
        headers['X-Redmine-API-Key'] = cst.SUBMITERS[cst.MANAGER]
360
        r = requests.get(cst.BASE_URL + '/entity_json/node/{}'.format(nid), headers=headers)
361
        page_json = json.loads(r.text)
362
        comments_json = page_json['comments']
363
        #If the issue has no comment, comments_json is a list, not a dict
364
        if comments_json:
365
            comments = list(comments_json.keys())
366
            return comments
367
        else:
368
            return list()
369

    
370
    def __iter__(self):
371
        self.__i = -1
372
        return self
373

    
374
    def __next__(self):
375
        self.__i += 1
376
        if self.__i >= len(self._issues):
377
            raise StopIteration
378
        return self._issues[self.__i]
379

    
380
    def __getitem__(self, index):
381
        return self._issues[index]
382

    
383
    def __len__(self):
384
        return len(self._issues)
385

    
386

    
387

    
388
class Redmine:
389
    """Main class.
390

391
    Allows the interaction with the program
392
    You can access any issue with the container notation.
393
    """
394
    def __init__(self, test=True):
395
        self.reset()
396
        self._test = test
397

    
398
    def reset(self):
399
        "Go to initial stage. All attributes are set to None"
400
        self._laundry = None
401
        self._headers = None
402
        self._headers_get = None
403

    
404
    def init(self, issues_file='issues.csv', x_redmine_api_key=cst.SUBMITERS[cst.MANAGER]):
405
        "Initialize the attribute for post uses"
406
        self._headers = cst.Headers
407
        self._headers['X-Redmine-API-Key'] = x_redmine_api_key
408
        self._laundry = Laundry(cst.LIST_TODO_CSV, self._test)
409

    
410
    def post(self, post_url=cst.URL_ISSUES_JSON):
411
        "Post all issues"
412
        nb_issues = len(self._laundry)
413
        i = 0
414
        for issue in self._laundry:
415
            i += 1
416
            print('Posting issues {} on {}'.format(i, nb_issues))
417
            issue.post(post_url, self._headers)
418

    
419
    def sweep(self):
420
        "Clean the redmine project of all issues."
421
        print('You are about to delete all issues on your redmine project.')
422
        ok = input('Do you wish to continue? (yes/no): ')
423

    
424
        if ok == 'yes':
425
            # Get the right headers
426
            self._headers_get = cst.Headers_GET
427
            self._headers_get['X-Redmine-API-Key'] = cst.SUBMITERS[cst.MANAGER]
428
            # Redmine give at maximum 100 issues. We may need to do it many times
429
            pass_number = 1
430
            while True:
431
                r = requests.get(cst.URL_ISSUES_JSON + '?status_id=*&limit=100',\
432
                                 headers=cst.Headers_GET)
433

    
434
                assert r.status_code == 200
435

    
436
                if not json.loads(r.text)['issues']: # There are no more issues to sweep
437
                    break
438
                print('Pass {}'.format(pass_number))
439
                taches_json = json.loads(r.text)['issues']
440
                # Print a nice completion percentage
441
                sys.stdout.flush()
442
                compt = 0
443
                print_progress(percentage(compt//len(taches_json)*100))
444
                for tache in taches_json:
445
                    tid = tache['id']
446
                    r = requests.delete(cst.URL_REDMINE + '/issues/{}.json'.format(tid),\
447
                                        headers=cst.Headers_GET)
448
                    compt += 1
449
                    print_progress(percentage(int(compt/len(taches_json)*100)))
450
                sys.stdout.write("\n")
451
                pass_number += 1
452
        else:
453
            print('Wise decision')
454

    
455
    def __getitem__(self, index):
456
        if self._laundry:
457
            return self._laundry[index]
458
        else:
459
            raise IndexError('Index out of range')
460

    
461

    
462

    
463

    
464
######## Main program
465
if __name__ == "__main__":
466
    redmine = Redmine(test=False)
467
    redmine.init()
468
    redmine.post()