Projet

Général

Profil

Révision e9b44dc1

Ajouté par Julien Enselme il y a plus de 10 ans

Version 2 de goto_redmine.py et constantes.py

Réécriture du script avec des classes afin de permettre une résolution
plus aisée des problèmes posés par l’ancien script :

  • Les captures d’écran ne sont pas présentes
  • Les liens vers les tâches sont cassés
  • L’ordre des commentaires n’est pas bon
  • Les informations sur les contributeurs et la date sont perdues Le script donne des informations sur la progression (ex post issue no 12 on 116)

goto_redmine.py et constantes.py ont été modifiés afin d’écrire
issues.csv et comments.csv qui permettent à fix-db de rétablir
directement dans la base de données les informations concernant les
contributeurs et la date de la contribution.

J'ai ajouté également des réponses de redmine lorsqu’on poste une tâche e$
commentaire pour information

Correction de constantes.py pour gianny

Voir les différences:

scripts_divers/migrer_taches_vers_redmine/constantes.py
1
#Dictionnaire des gens dont on a la clé API
2
#id: clé
3
SUBMITERS = {'jenselme': '', 'admin': 'a7630a1244be353424cc0f56a49657c3fa9dbcc6'}
4
MANAGER_TEST = 'admin'
5
MANAGER = 'jenselme'
1
######## Dict of people who have an api-keys
2
#id: key
3
SUBMITERS = {'jenselme': 'b3ce5345b1444b66f05f6a143ffec6f401e8f17e', 'admin': 'a7630a1244be353424cc0f56a49657c3fa9dbcc6'}
4
MANAGER = 'admin' # Can post issues
5
#MANAGER = 'jenselme'
6 6

  
7
#les entêtes des requêtes http
7

  
8
######## HTTP headers
8 9
Headers = {'content-type': 'application/json', 'X-Redmine-API-Key': ''}
9 10
Headers_GET = {'X-Redmine-API-Key': ''}
10 11

  
11
#là où on poste les tâches
12
URL_POST_ISSUES = 'https://forge.centrale-marseille.fr/issues'
13
#La liste des tâches
14
URL_ISSUES_JSON_TEST = 'http://localhost/redmine/projects/test/issues.json'
15
URL_ISSUES_JSON = 'https://forge.centrale-marseille.fr/projects/clubdrupal/issues.json'
16
#l’url du projet
17
URL_PROJECT_TEST = 'http://localhost/redmine/projects/test'
18
URL_PROJECT = 'https://forge.centrale-marseille.fr/projects/clubdrupal'
19
URL_REDMINE_TEST = 'http://localhost/redmine'
20
URL_REDMINE = 'http://forge.centrale-marseille.fr'
21

  
22
#là où sont les tâches
23
LIST_TODO = 'http://localhost/portail/liste-tache'
24

  
25
#url de base de l’emplacement du contenu
26
BASE_URL = 'http://localhost/portail'
27
PROJECT_ID = 30
12

  
13
######## Base URL of where to post the issues
14
URL_ISSUES = 'http://debian/redmine/issues'
15
#URL_POST_ISSUES = 'https://forge.centrale-marseille.fr/issues'
16

  
17

  
18
######## Issues list in redmine of a give project
19
URL_ISSUES_JSON = 'http://debian/redmine/projects/test/issues.json'
20
#URL_ISSUES_JSON = 'https://forge.centrale-marseille.fr/projects/clubdrupal/issues.json'
21

  
22
######## Project URL
23
URL_PROJECT = 'http://debian/redmine/projects/test'
24
#URL_PROJECT = 'https://forge.centrale-marseille.fr/projects/clubdrupal'
25
URL_REDMINE = 'http://debian/redmine'
26
#URL_REDMINE = 'http://forge.centrale-marseille.fr'
27

  
28

  
29
######## Issues location in drupal (node id in a csv file to be parsed)
30
LIST_TODO = 'http://debian/portail/liste-tache'
31
LIST_TODO_CSV = 'http://debian/portail/liste_tache_csv'
32

  
33

  
34
######## Base url of drupal
35
BASE_URL = 'http://debian/portail'
36

  
37

  
38
######## Various id
39
PROJECT_ID = 1
40
#PROJECT_ID = 30
28 41
TRACKER_ID = 2
42
USER_ID = {'ftorregrosa': 45, 'Ismaeil ABOULJAMAL': 44, 'iabouljamal': 44,\
43
           'Nono LiNux': 47, 'nlehuby': 47, 'gfranchi': 78, 'assos': 78, '82': 78, 'dgeo': 22,\
44
           'Jean': 48,'jfeutry': 48, 'adeprey': 49,'jpennec': 46, 'gt': 42, 'jenselme': 43,
45
           '1': 44, '176': 43, '123': 48, '114': 49,\
46
           '99': 46, '97': 45, '20': 47, '16': 42, '9': 22}
29 47

  
30 48

  
31
########## dictionnaires de correspondance
49
######## Dict to translate drupal's fields into redmine ids
32 50
DONE_RATIO = {'En pause': 50, 'À commencer': 0, 'Entamée': 20, 'Bien avancée': 80, 'Terminée (success)': 100, 'Fermée (won\'t fix)': 100}
33 51
PRIORITY = {'5 - Très basse': 3, '4 - Basse': 3, '3 - Moyenne': 4, '2 - Haute': 5,\
34 52
            '1 - Très haute': 6, 'Très basse': 3, 'Basse': 3, 'Moyenne': 4,\
35 53
            'Haute': 5, 'Très haute':6, '0': 3, '1': 3, '2': 4, '3': 5, '4': 6}
36 54
STATUS = {'En cours': 2, 'Fermée': 5, 'Rejetée': 6, 'En pause': 7}
37
#NB sur le portail, on a les équivalences suivantes
38
#pour le champ version de drupal : 17 : drupal6, 18 : drupal7
55
#NB: sur le portail, on a les équivalences suivantes
56
# For drupal_version_field : 17 : drupal6, 18 : drupal7
39 57
DRUPAL_VERSION = {'17': 2, '18': 1}
58

  
59

  
60
###### Other
61
REGEXP_NAME = 'http://debian/portail/content/t%C3%A2che/(.*)'
scripts_divers/migrer_taches_vers_redmine/fix-db.pl
1
# In order to complete the migration to Redmine, we need to preserve date
2
# and username. To do this, we need to fix the database.
3

  
4
use strict;
5
use warnings;
6
use DBI;
7

  
8
my $db		= 'redmine_default';
9
my $server	= 'localhost'; # Il est possible de mettre une adresse IP
10
my $user        = 'root';      # identifiant
11
my $passwd	= 'tata';
12
my $port	= '';
13

  
14

  
15
##### Create DB connection
16
my $dbh = DBI->connect( "DBI:mysql:database=$db;host=$server;port=$port", $user, $passwd, { RaiseError => 1, } ) or die "Connection impossible à la base de données $db !\n $! \n $@\n$DBI::errstr";
17

  
18

  
19

  
20
###### Initialisation of hashes needed to fix URL
21
open(my $fix_url_issues_csv, '<', 'fix_url_issues.csv') or die "Couldn't open fix_url_issues.csv";
22
my %name2iid = ();
23
my %nid2iid = ();
24
my %cid2iid = ();
25
my %cid2post_nb = ();
26

  
27
while (my $line = <$fix_url_issues_csv>)
28
  {
29
    chomp $line;
30
    my ($nid, $name, $iid) = split ",", $line;
31
    $name2iid{$name} = $iid;
32
    $nid2iid{$nid} = $iid;
33
  }
34
close $fix_url_issues_csv;
35

  
36
open(my $fix_url_comments_csv, '<', 'fix_url_comments.csv') or die "Couldn't open fix_url_comments.csv";
37
while (my $line = <$fix_url_comments_csv>)
38
  {
39
    chomp $line;
40
    my ($cid, $iid, $post_nb) = split ",", $line;
41
    $cid2iid{$cid} = $iid;
42
    $cid2post_nb{$cid} = $post_nb;
43
  }
44
close $fix_url_comments_csv;
45

  
46

  
47

  
48
###### Definition of functions
49
sub fix_issues_link {
50
    my $text = $_[0];
51

  
52
    # Find all links in text
53
    my @links = ($text =~ m/:http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)(?:content\/t%C3%A2che|node)\/(.*)/g);
54

  
55
    if ( @links )
56
    {
57
	foreach (@links)
58
	{
59
	    print $_ ."\n";
60
	    $text =~ s/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)(content\/t%C3%A2che|node)\/.*":http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)(content\/t%C3%A2che|node)\/$_/#$cid2iid{$_}#note-$cid2post_nb{$_}/g;
61
	}
62
    }
63
    return $text;
64
}
65

  
66
sub fix_comments_link {
67
    my $text = $_[0];
68

  
69
    # Find all links in text
70
    my @links = $text =~ m/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)comment\/.*"/g;
71

  
72
     # Foreach link, get comment cid.
73
    my @cids = ();
74
    if (@links)
75
    {
76
	foreach (@links)
77
	{
78
	    my $cid =~ s/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)comment\/(.*)#.*"/$1/g;
79
	    push @cids, $cid;
80
	}
81

  
82
	foreach (@cids)
83
	{
84
	    $text =~ s/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)comment\/$_#$_":http:\/\/assos\.centrale-marseille\.fr\/comment\/$_#$_/#$name2iid{$_}/g;
85
	}
86
    }
87

  
88
    return $text;
89
}
90

  
91

  
92

  
93
###### Update issues
94
my $sql_update_issue =<<"SQL";
95
UPDATE issues
96
  SET author_id = ?, created_on = ?
97
  WHERE  id = ?
98
SQL
99

  
100
my $sql_fix_links_issue =<<"SQL";
101
UPDATE issues
102
    SET description = ?
103
    WHERE id = ?
104
SQL
105

  
106
my $req_update_issue = $dbh->prepare($sql_update_issue);
107
my $req_fix_links_issue = $dbh->prepare($sql_fix_links_issue);
108

  
109
# Reading file
110
open(my $issues, '<issues.csv') or die "Couldn't open issues.txt\n";
111

  
112
while (my $line = <$issues>)
113
  {
114
    # Update author and date of creation
115
    chomp $line;
116

  
117
    my ($iid, $author_id, $created_on, $nid, $name) = split ",", $line;
118

  
119
    $req_update_issue->execute($author_id, $created_on, $iid);
120

  
121
    ## Fix links
122
    # issues
123
    my $select_description = "SELECT description FROM issues WHERE id = 1569";
124

  
125
    my ($description) = $dbh->selectrow_array($select_description);
126

  
127
    $description = fix_issues_link($description);
128

  
129
    die "stop";
130

  
131
    print $description . "\n";
132
#    print $links[0];
133
#    print $name2iid{$links[0]};
134
    # Comments
135
#    $description = fix_comments_link($description);
136

  
137
#    $req_fix_links_issue->execute($description, $iid);
138
  }
139

  
140
$req_update_issue->finish;
141
$req_fix_links_issue->finish;
142

  
143

  
144
###### Update journals
145
my $sql_get_id =<<"SQL";
146
SELECT id FROM journals WHERE journalized_id = ?
147
ORDER BY created_on ASC
148
SQL
149

  
150
my $sql_update_journals =<<"SQL";
151
UPDATE journals
152
SET created_on = ?, user_id = ?
153
WHERE id = ?
154
SQL
155

  
156
my $sql_updated_on =<<"SQL";
157
UPDATE issues SET updated_on = ?
158
WHERE id = ?
159
SQL
160

  
161
my $sql_fix_links_comment =<<"SQL";
162
UPDATE journals
163
    SET notes = ?
164
    WHERE id = ?
165
SQL
166

  
167
my $req_get_id = $dbh->prepare($sql_get_id);
168
my $req_update_journals = $dbh->prepare($sql_update_journals);
169
my $req_updated_on = $dbh->prepare($sql_updated_on);
170
my $req_fix_links_comment = $dbh->prepare($sql_fix_links_comment);
171

  
172
# Reading file.
173
open(my $comments, '<comments.csv') or die "Couldn't open comments.txt\n";
174

  
175
# We must remember current iid in order to fetch result or execute query: req_get_id return
176
# more than one value
177
my $current_iid = -1;
178

  
179
while (my $line = <$comments>)
180
  {
181
    chomp $line;
182

  
183
    my ($iid, $user_id, $created_on) = split ",", $line;
184

  
185
    # We get the id of the comment.
186
    if ($current_iid != $iid)
187
    {
188
	$current_iid = $iid;
189
	$req_get_id->execute($iid);
190
    }
191

  
192
    my ($id) = $req_get_id->fetchrow_array;
193

  
194
    # We do the update.
195
    $req_update_journals->execute($created_on, $user_id, $id);
196
    $req_updated_on->execute($created_on, $iid);
197

  
198
    ## Fix links.
199
    # issues
200
    my $select_notes = "SELECT notes FROM journals WHERE id = $id";
201
    my ($notes) = $dbh->selectrow_array($select_notes);
202
#    $notes = fix_issues_link($notes);
203
    # Comments
204
#    $notes = fix_comments_link($notes);
205

  
206
#    $req_fix_links_comment->execute($notes, $id);
207
  }
208

  
209
$req_get_id->finish;
210
$req_update_journals->finish;
211
$req_updated_on->finish;
212

  
213
####### Close the db connection.
214
$dbh->disconnect;
scripts_divers/migrer_taches_vers_redmine/goto_redmine.py
1
"""
2
Pandoc est requis pour convertir le html en textile !
1
#!/usr/bin/env python3
2

  
3
"""This script can migrate issues from drupal to redmine.
4

  
5
It is design be launched with the -i option: most variables you may need for
6
debugging are easily accessible from interpreter (like HTTP status codes).
7
We make some assertion based on HTTP status code. Here are the codet you may
8
wish to know:
9
- 200: OK
10
- 201: Created
11
- 404: Not found
12
- 403: Forbidden
13
- 500: Internal error
14
Redmine gives you more intel in its response. Read them!
15
Pandoc is required to convert html syntax to textile syntax
3 16
"""
4 17

  
5
import url_parser  #permet de connaître les id des taches
6
import urllib.request #permet de récupérer une page web
7
import httplib2 #pour faire des requêtes http
18
import requests
8 19
import json
9
import re #pour les expressions régulières
10
import os #pour pouvoir faire appel à pandoc (commande system)
11

  
12
######## NB : cid : comment id, nid : node id, urls : urls des tâches
13

  
14
#Dictionnaire des gens dont on a la clé API
15
#id: clé
16
SUBMITERS = {'jenselme': '464c7c05b9bb53fb136092f1b9807ad91ec51321'}
17

  
18
#les entêtes des requêtes POST et PUT
19
Headers = {'content-type': 'application/json', 'X-Redmine-API-Key': ''}
20

  
21
#là où on poste les tâches
22
URL = 'https://forge.centrale-marseille.fr/issues'
23

  
24
#là où sont les tâches
25
LIST_TODO = 'http://localhost/portail/liste-tache'
26

  
27
#url de base de l’emplacement du contenu
28
BASE_URL = 'http://localhost/portail'
29
PROJECT_ID = 30
30
TRACKER_ID = 2
31

  
32

  
33
########## dictionnaires de correspondance
34
DONE_RATIO = {'En pause': 50, 'À commencer': 0, 'Entamée': 20, 'Bien avancée': 80, 'Terminée (success)': 100, 'Fermée (won\'t fix)': 100}
35
PRIORITY = {'5 - Très basse': 3, '4 - Basse': 3, '3 - Moyenne': 4, '2 - Haute': 5, '1 - Très haute': 6,\
36
        'Très basse': 3, 'Basse': 3, 'Moyenne': 4, 'Haute': 5, 'Très haute':6,\
37
        '0': 3, '1': 3, '2': 4, '3': 5, '4': 6}
38
STATUS = {'En cours': 2, 'Fermée': 5, 'Rejetée': 6, 'En pause': 7}
39
#NB sur le portail, on a les équivalences suivantes
40
#pour le champ version de drupal : 17 : drupal6, 18 : drupal7
41
DRUPAL_VERSION = {'17': 2, '18': 1}
42

  
43
def give_api_key(submiter):
44
    "Donne la clé API de submiter ou celle de jenselme si c’est la seule"
45
    if submiter in SUBMITERS:
46
        return SUBMITERS[submiter]
47
    else:
48
        return  SUBMITERS['jenselme']
49

  
50

  
51
def give_comments_ids(nid):
52
    "permet de récupérer les id des commentaires de la tâche nid"
53
    page = urllib.request.urlopen(BASE_URL + '/entity_json/node/' + nid).read()
54
    page_json = json.loads(page.decode('utf-8'))
55
    comments_json = page_json['comments']
56
    #S’il n’y a pas de commentaire, comments_json est une liste vide et pas un dictionnaire
57
    if comments_json:
58
        comments = list(comments_json.keys())
59
        comments.sort() #ce sont les clés d’un dictionnaire. Pas d’ordre à priori
60
        return comments
61
    else:
62
        return list()
63

  
64

  
65
def give_comments(cids):
66
    "Donne la liste du texte des commentaires pour chaque cid in cids"
67
    comments = list()
68
    for cid in cids:
69
        comment = urllib.request.urlopen(BASE_URL + '/comment/' + cid + '.json').read()
70
        comments.append(json.loads(comment.decode('utf-8')))
71
    return comments
72

  
73

  
74
def format(txt):
75
    "prend le texte en html et le renvoie en textile"
20
import sys
21
import datetime
22
import os #we need system() to call pandoc
23
import re
24

  
25
import constantes as cst
26

  
27
######## Global variables
28
REGEXP_FIND_IMG = re.compile('!/.*!')
29
REGEXP_NAME_IMG = re.compile('!.*/(.*)!')
30

  
31
######## Common functions
32

  
33
def handle_image(txt):
34
    "Images are not posted automatically. There are only few of them.\
35
    We just format the text with the correct textile syntax and when we post them,\
36
    we will add comment and node id into a file. They should be attached to the\
37
    correct issue afterwards"
38
    has_image = False
39
    # In textile, images are between !
40
    images = REGEXP_FIND_IMG.findall(txt)
41
    print(images)
42
    if images:
43
        has_image = True
44
        for image in images:
45
            img_name = REGEXP_NAME_IMG.sub(r'!\1!', image)
46
            txt.replace(image, img_name)
47
    return txt, has_image
48

  
49
def html2textile(txt):
50
    "Convert a txt from html to textile using pandoc"
51
    # We remove line breaks and tabs, otherwise the conversion doesn't work properly
76 52
    txt.replace('\n', '')
77 53
    txt.replace('\t', '')
54
    # pandoc can only manipulates files
78 55
    with open('tmp.html', 'w') as f:
79 56
        f.write(txt)
80 57
    os.system('pandoc -f html tmp.html -t textile -o tmp.textile')
81 58
    with open('tmp.textile', 'r') as f:
82 59
        txt = f.read()
83
    return txt
84

  
85

  
86
def give_redmine_status_id(tache):
87
    drupal_status = ''
88
    for elt in tache['field_avancement']:
89
        if "Terminée" in elt:
90
            drupal_status = 'Fermée'
91
            break
92
        elif "Fermée" in elt:
93
            drupal_status = 'Rejetée'
94
            break
95
        elif "pause" in elt:
96
            drupal_status = 'En pause'
97
            del elt
98
            break
99
    if not drupal_status:
100
        drupal_status = 'En cours'
101
    return STATUS[drupal_status]
102

  
103

  
104
def give_redmine_issue(tache):
105
    issue = dict()
106
    issue['project_id'] = PROJECT_ID
107
    issue['tracker_id'] = TRACKER_ID
108
    issue['subject'] = tache['title']
109
    issue['description'] = format(tache['body']['value'])
110
    #de temps en temps, le champ priorité est vide. On met 'Normale' dans ce cas
111
    if tache['field_prioritaecute']:
112
        issue['priority_id'] = PRIORITY[tache['field_prioritaecute']]
113
    else:
114
        issue['priority_id'] = PRIORITY['3 - Moyenne']
115
    if tache['field_avancement']:
116
        issue['done_ratio'] = DONE_RATIO[tache['field_avancement'][0]]
60
    # Cleaning temporary files
61
    os.remove('tmp.html')
62
    os.remove('tmp.textile')
63
    return handle_image(txt)
64

  
65
def egalise(string, length):
66
    "Make the length of string equals to length if shorter"
67
    if len(string) < length:
68
        return ' '*(length - len(string)) + string
117 69
    else:
118
        issue['done_ratio'] = DONE_RATIO['À commencer']
119
    issue['status_id'] = give_redmine_status_id(tache)
120
    issue['fixed_version_id'] = DRUPAL_VERSION[tache['taxonomy_vocabulary_8']['id']]
121
    return issue
122

  
123

  
124
######### Main
125

  
126
nids, urls = url_parser.give_json_urls(LIST_TODO, BASE_URL)
70
        return string
71

  
72
def percentage(integer):
73
    "Converts integer into a string used to indicate the percentage of completion\
74
    of a command"
75
    string = str(integer)
76
    string = egalise(string, 3)
77
    string = 'Completion: ' + string + '%'
78
    return string + '\b'*len(string)
127 79

  
128
h = httplib2.Http()
80
def print_progress(str):
81
    sys.stdout.write(str)
82
    sys.stdout.flush()
129 83

  
130
for post_url in urls:
131
    nid = nids[urls.index(post_url)]
132
    print(nid)
133
    tache_json = urllib.request.urlopen(post_url)
134
    tache_drupal = json.loads(tache_json.read().decode('utf-8'))
84
def format_date(timestamp):
85
    str_timestamp = float(timestamp)
86
    date = datetime.datetime.fromtimestamp(str_timestamp)
87
    return date.strftime('%Y-%m-%d %H:%M:%S')
135 88

  
136
    cids = give_comments_ids(nid)
137
    comments_drupal = give_comments(cids)
138

  
139
    issue = {}
140
    issue['issue'] = give_redmine_issue(tache_drupal)
141
    data = json.dumps(issue)
142

  
143
    Headers['X-Redmine-API-Key'] = SUBMITERS['jenselme']
144

  
145
    resp, content = h.request(URL + '.json', 'POST', body=data, headers=Headers)
146

  
147
    #on récupère l’issue id pour savoir où poster les commentaires
148
    iid = re.findall(r',"id":([0-9]*),', content.decode('utf-8'))[0]
149

  
150
    #on a besoin de l’url à laquelle on met les commentaires, pour changer le status
151
    put_url = URL + '/' + iid + '.json'
152
    for index, comment in enumerate(comments_drupal):
153
        submiter = comment['name']  #le premier est celui qui a soumis le node
154
        Headers['X-Redmine-API-Key'] = give_api_key(submiter)
155
        #si la personne n’a pas sa clé, on modifie le commentaire
156
        comment_body = format(comment['comment_body']['value'])
157
        if not submiter in SUBMITERS:
158
            comment_body = "_{}_ a dit que :\n\n{}".format(submiter, comment_body)
159
        update = {}
160
        update['issue'] = {'notes': comment_body}
161
        data = json.dumps(update)
162
        h.request(put_url, 'PUT', body=data, headers=Headers)
163 89

  
164
    #Les taches sont crées avec le status nouveau peu importe ce qu’il y a dans le json
165
    #on modifie le status après coup
166
    update_status = {'issue': {'status_id': issue['issue']['status_id']}}
167
    data = json.dumps(update_status)
168
    h.request(put_url, 'PUT', body=data, headers=Headers)
90

  
91
######## Definition of classes
92

  
93

  
94
class Comment:
95
    """Represents a drupal comment
96
    """
97

  
98
    def __init__(self, cid, author, post_date, content, has_img):
99
        self._cid = cid # comment id in drupal
100
        self._author = author
101
        self._post_date = post_date
102
        self._content = content
103
        # json representation, to be posted in redmine
104
        self._update = {'issue': {'notes': self._content }}
105
        self._update_json = json.dumps(self._update)
106
        self._resp = None #will be used to store the put response
107
        self._has_img = has_img
108

  
109
    def post(self, url, headers, iid, post_nb):
110
        "Post the comment to url with headers (required for authentication)"
111
        self._resp = requests.put(url, headers=headers, data=self._update_json)
112
        assert self._resp.status_code == 200
113

  
114
        # We write iid,author_id,created_on in comments.csv
115
        with open('comments.csv', 'a', encoding='utf8') as comments_csv:
116
            comments_csv.write('{},{},{}\n'.\
117
                        format(iid, cst.USER_ID[self._author], self._post_date))
118
        with open('fix_url_comments.csv', 'a', encoding='utf8') as fix_url_csv:
119
            fix_url_csv.write('{},{},{}\n'.format(self._cid, iid, post_nb))
120

  
121
    @property
122
    def post_date(self):
123
        return self._post_date
124

  
125
    @property
126
    def resp(self):
127
        return self._resp
128

  
129
    @property
130
    def cid(self):
131
        return self._cid
132

  
133
    @property
134
    def has_img(self):
135
        return self._has_img
136

  
137

  
138

  
139
class Updates:
140
    """Represents all the comments of a task
141
    """
142

  
143
    def __init__(self, comments):
144
        self._comments = comments
145

  
146
    def sort(self):
147
        "Sort all the updates by date of creation"
148
        sorted_date = False
149
        while not sorted_date:
150
            sorted_date = True
151
            i = 0
152
            while i < len(self._comments) - 1:
153
                if self._comments[i].post_date > self._comments[i + 1].post_date:
154
                    self._comments[i], self._comments[i + 1] = self._comments[i + 1],\
155
                                                               self._comments[i]
156
                    sorted_date = False
157
                i += 1
158

  
159
    def __getitem__(self, index):
160
        return self._comments[index]
161

  
162
    def __len__(self):
163
        return len(self._comments)
164

  
165
    def __iter__(self):
166
        self.__i = -1
167
        return self
168

  
169
    def __next__(self):
170
        self.__i += 1
171
        if self.__i >= len(self._comments) or len(self._comments) == 0:
172
            raise StopIteration
173
        return self._comments[self.__i]
174

  
175

  
176

  
177
class Issue:
178
    """Represents a drupal issue
179
    """
180

  
181
    def __init__(self, nid, comments):
182
        self._nid = nid #node id
183
        self._iid = None #issue id, unknown until creation
184
        self._resp = None #will be used to store the response of requests.post
185
        self._comments = Updates(comments)
186
        self._comments.sort()
187
        self._issue = self.give_redmine_issue(nid) #the actual content, it's a dict
188

  
189
    def give_redmine_status_id(self, node):
190
        "Translate the drupal status field to an integer representing the\
191
        redmine status id"
192
        drupal_status = ''
193
        for elt in node['field_avancement']:
194
            if "Terminée" in elt:
195
                drupal_status = 'Fermée'
196
                break
197
            elif "Fermée" in elt:
198
                drupal_status = 'Rejetée'
199
                break
200
            elif "pause" in elt:
201
                drupal_status = 'En pause'
202
                del elt
203
                break
204
        if not drupal_status:
205
            drupal_status = 'En cours'
206
        return cst.STATUS[drupal_status]
207

  
208
    def give_redmine_issue(self, nid):
209
        "Uses the nid to find the node and converts its content to something\
210
        redmine can understand. Read examples for more intels"
211
        node_json = requests.get(cst.BASE_URL + '/node/{}.json'.format(nid)).text
212
        node = json.loads(node_json)
213
        issue = dict()
214
        issue['project_id'] = cst.PROJECT_ID
215
        issue['tracker_id'] = cst.TRACKER_ID
216
        issue['subject'] = node['title']
217
        issue['description'], self._has_img = html2textile(node['body']['value'])
218
        # We get the name of the node
219
        self._name = re.findall(cst.REGEXP_NAME, node['url'])[0]
220
        # field_prioritaecute can be empty. We then assume it is normal
221
        if node['field_prioritaecute']:
222
            issue['priority_id'] = cst.PRIORITY[node['field_prioritaecute']]
223
        else:
224
            issue['priority_id'] = cst.PRIORITY['3 - Moyenne']
225
        # field_avancement can be empty. We then assume it is to be started
226
        if node['field_avancement']:
227
            issue['done_ratio'] = cst.DONE_RATIO[node['field_avancement'][0]]
228
        else:
229
            issue['done_ratio'] = cst.DONE_RATIO['À commencer']
230
        # Status id = open, fix, closed…
231
        issue['status_id'] = self.give_redmine_status_id(node)
232
        issue['fixed_version_id'] = cst.DRUPAL_VERSION[node['taxonomy_vocabulary_8']['id']]
233
        issue['created'] = format_date(node['created'])
234
        issue['author_id'] = node['author']['id']
235
        # Do we have attached files?
236
        if node['field_fichier']:
237
            self._has_files = True
238
        else:
239
            self._has_files = False
240
        return issue
241

  
242
    def post(self, url, headers):
243
        "Post the comment to url with headers (required for authentication)"
244
        issue = {'issue': self._issue}
245
        data = json.dumps(issue)
246

  
247
        self._resp = requests.post(url, headers=headers, data=data)
248

  
249
        assert self._resp.status_code == 201
250

  
251
        resp_json = json.loads(self._resp.text)
252
        self._iid = resp_json['issue']['id']
253

  
254
        # We write iid,author_id,created_on in issues.csv
255
        with open('issues.csv', 'a', encoding='utf8') as issues_csv:
256
            author_id = self._issue['author_id']
257
            redmine_author_id = cst.USER_ID[author_id]
258
            created_on = self._issue['created']
259
            issues_csv.write('{},{},{}\n'.\
260
                             format(self._iid, redmine_author_id, created_on))
261
        with open('fix_url_issues.csv', 'a') as fix_url:
262
            fix_url.write('{},{},{}\n'.format(self._nid, self._name, self._iid))
263

  
264
        # We post comments
265
        nb_comments = len(self._comments)
266
        i = 0
267
        for comment in self._comments:
268
            i += 1
269
            put_url =  cst.URL_ISSUES + '/{}.json'.format(self._iid)
270
            comment.post(put_url, headers, self._iid, i)
271
            print_progress(percentage(i//nb_comments*100))
272

  
273
        # We take care of images and files
274
        self.handle_image()
275
        self.handle_files()
276

  
277
    def handle_files(self):
278
        if self._has_files:
279
            with open('has_files.txt', 'a', encoding='utf8') as has_files_file:
280
                has_files_file.write('{}\n'.format(self._nid))
281

  
282
    def handle_image(self):
283
        for comment in self._comments:
284
            self._has_img = comment.has_img or self._has_img
285
        if self._has_img:
286
            with open('has_img.txt', 'a', encoding='utf8') as has_img_file:
287
                has_img_file.write('{}\n'.format(self._nid))
288

  
289
    @property
290
    def resp(self):
291
        return self._resp
292

  
293
    @property
294
    def comments_resp(self):
295
        resps = dict()
296
        for comment in self._comments:
297
            resps[comment.cid] = comment.resp
298
        return resps
299

  
300

  
301

  
302
class Laundry:
303
    """Contains all issues and has methods to perform the migration
304

  
305
    Iteration of this object traverses all issues.
306
    You can access any issue with the container notation
307
    """
308

  
309
    def __init__(self, url, test=True):
310
        self._issues = self.give_issues(url, test)
311

  
312
    def give_issues(self, url, test):
313
        "Returns a list of all issues"
314
        issues = []
315
        r = requests.get(url)
316

  
317
        assert r.status_code == 200
318

  
319
        # 1st element is 'Nid', and last is ''
320
        nids = r.text.split('\r\n')[1:-1]
321
        # for test, we only use 3 issues (faster)
322
        if test:
323
            nids = nids[:3]
324
        self.nb_task = len(nids)
325
        i = 0
326
        for nid in nids:
327
            i += 1
328
            comments = self.give_comments(nid)
329
            print('Fetching issue no {} on {}'.format(i, self.nb_task))
330
            issues.append(Issue(nid, comments))
331
        return issues
332

  
333
    def give_comments(self, nid):
334
        "Returns the list of all comments of a node"
335
        cids, comments_json = self.give_comments_json(nid)
336
        comments = []
337
        i = 0
338
        nb_comments = len(comments_json)
339
        for comment_json in comments_json:
340
            author = comment_json['name']
341
            post_date = format_date(comment_json['created'])
342
            content, has_img = html2textile(comment_json['comment_body']['value'])
343
            comments.append(Comment(cids[comments_json.index(comment_json)], author, post_date, content, has_img))
344
            i += 1
345
            print_progress(percentage(i//nb_comments*100))
346
        return comments
347

  
348
    def give_comments_json(self, nid):
349
        "Get the raw json version of the drupal comment"
350
        cids = self.give_comments_ids(nid)
351
        comments = list()
352
        for cid in cids:
353
            comment = requests.get(cst.BASE_URL + '/comment/' + cid + '.json')
354
            comments.append(json.loads(comment.text))
355
        return cids, comments
356

  
357
    def give_comments_ids(self, nid):
358
        "Get the cid (comnments id) for a node"
359
        headers = cst.Headers_GET
360
        headers['X-Redmine-API-Key'] = cst.SUBMITERS[cst.MANAGER]
361
        r = requests.get(cst.BASE_URL + '/entity_json/node/{}'.format(nid), headers=headers)
362
        page_json = json.loads(r.text)
363
        comments_json = page_json['comments']
364
        #If the issue has no comment, comments_json is a list, not a dict
365
        if comments_json:
366
            comments = list(comments_json.keys())
367
            return comments
368
        else:
369
            return list()
370

  
371
    def __iter__(self):
372
        self.__i = -1
373
        return self
374

  
375
    def __next__(self):
376
        self.__i += 1
377
        if self.__i >= len(self._issues):
378
            raise StopIteration
379
        return self._issues[self.__i]
380

  
381
    def __getitem__(self, index):
382
        return self._issues[index]
383

  
384
    def __len__(self):
385
        return len(self._issues)
386

  
387

  
388

  
389
class Redmine:
390
    """Main class.
391

  
392
    Allows the interaction with the program
393
    You can access any issue with the container notation.
394
    """
395
    def __init__(self, test=True):
396
        self.reset()
397
        self._test = test
398

  
399
    def reset(self):
400
        "Go to initial stage. All attributes are set to None"
401
        self._laundry = None
402
        self._headers = None
403
        self._headers_get = None
404

  
405
    def init(self, issues_file='issues.csv', x_redmine_api_key=cst.SUBMITERS[cst.MANAGER]):
406
        "Initialize the attribute for post uses"
407
        self._headers = cst.Headers
408
        self._headers['X-Redmine-API-Key'] = x_redmine_api_key
409
        self._laundry = Laundry(cst.LIST_TODO_CSV, self._test)
410

  
411
    def post(self, post_url=cst.URL_ISSUES_JSON):
412
        "Post all issues"
413
        nb_issues = len(self._laundry)
414
        i = 0
415
        for issue in self._laundry:
416
            i += 1
417
            print('Posting issues {} on {}'.format(i, nb_issues))
418
            issue.post(post_url, self._headers)
419

  
420
    def sweep(self):
421
        "Clean the redmine project of all issues."
422
        print('You are about to delete all issues on your redmine project.')
423
        ok = input('Do you wish to continue? (yes/no): ')
424

  
425
        if ok == 'yes':
426
            # Get the right headers
427
            self._headers_get = cst.Headers_GET
428
            self._headers_get['X-Redmine-API-Key'] = cst.SUBMITERS[cst.MANAGER]
429
            # Redmine give at maximum 100 issues. We may need to do it many times
430
            pass_number = 1
431
            while True:
432
                r = requests.get(cst.URL_ISSUES_JSON + '?status_id=*&limit=100',\
433
                                 headers=cst.Headers_GET)
434

  
435
                assert r.status_code == 200
436

  
437
                if not json.loads(r.text)['issues']: # There are no more issues to sweep
438
                    break
439
                print('Pass {}'.format(pass_number))
440
                taches_json = json.loads(r.text)['issues']
441
                # Print a nice completion percentage
442
                sys.stdout.flush()
443
                compt = 0
444
                print_progress(percentage(compt//len(taches_json)*100))
445
                for tache in taches_json:
446
                    tid = tache['id']
447
                    r = requests.delete(cst.URL_REDMINE + '/issues/{}.json'.format(tid),\
448
                                        headers=cst.Headers_GET)
449
                    compt += 1
450
                    print_progress(percentage(int(compt/len(taches_json)*100)))
451
                sys.stdout.write("\n")
452
                pass_number += 1
453
        else:
454
            print('Wise decision')
455

  
456
    def __getitem__(self, index):
457
        if self._laundry:
458
            return self._laundry[index]
459
        else:
460
            raise IndexError('Index out of range')
461

  
462

  
463

  
464

  
465
######## Main program
466
if __name__ == "__main__":
467
    redmine = Redmine(test=False)
468
    redmine.init()
469
    redmine.post()
scripts_divers/migrer_taches_vers_redmine/output_post.html
1
* About to connect() to localhost port 80 (#0)
2
*   Trying ::1...
3
* connected
4
* Connected to localhost (::1) port 80 (#0)
5
> POST /redmine/issues.json HTTP/1.1
6
> User-Agent: curl/7.26.0
7
> Host: localhost
8
> Accept: */*
9
> Content-Type: application/json
10
> X-Redmine-API-Key: a7630a1244be353424cc0f56a49657c3fa9dbcc6
11
> Content-Length: 255
12
> 
13
* upload completely sent off: 255 out of 255 bytes
14
* additional stuff not fine transfer.c:1037: 0 0
15
* HTTP 1.1 or later with persistent connection, pipelining supported
16
< HTTP/1.1 201 Created
17
< Date: Tue, 02 Jul 2013 10:18:03 GMT
18
< Server: Apache/2.2.22 (Debian)
19
< X-Powered-By: Phusion Passenger (mod_rails/mod_rack) 3.0.13
20
< Cache-Control: no-cache
21
< X-Runtime: 173
22
< Set-Cookie: autologin=; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT
23
< Set-Cookie: _redmine_default=BAh7BjoPc2Vzc2lvbl9pZCIlOWMyMDE3ZjUyOWMwYzFhNzkzODk3NzkwMzcxMDI0Njg%3D--53d46d1fb4e9684381a35ad0951b669869dde966; path=/redmine; HttpOnly
24
< Location: http://localhost/redmine/issues/7
25
< Content-Length: 433
26
< Status: 201
27
< Content-Type: application/json; charset=utf-8
28
< 
29
* Connection #0 to host localhost left intact
30
{"issue":{"status":{"name":"New","id":1},"done_ratio":100,"subject":"tache 8","updated_on":"2013/07/02 11:18:03 +0100","tracker":{"name":"Feature","id":2},"spent_hours":0.0,"description":"J'\u00e9cris plein de chose\r\nPour ma premi\u00e8re tache","start_date":"2013/07/02","author":{"name":"Redmine Admin","id":1},"created_on":"2013/07/02 11:18:03 +0100","project":{"name":"test","id":1},"id":7,"priority":{"name":"Urgent","id":6}}}* Closing connection #0
scripts_divers/migrer_taches_vers_redmine/output_put.html
1
* About to connect() to localhost port 80 (#0)
2
*   Trying ::1...
3
* connected
4
* Connected to localhost (::1) port 80 (#0)
5
> PUT /redmine/issues/6.json HTTP/1.1
6
> User-Agent: curl/7.26.0
7
> Host: localhost
8
> Accept: */*
9
> Content-Type: application/json
10
> X-Redmine-API-Key: a7630a1244be353424cc0f56a49657c3fa9dbcc6
11
> Content-Length: 52
12
> 
13
* upload completely sent off: 52 out of 52 bytes
14
* additional stuff not fine transfer.c:1037: 0 0
15
* HTTP 1.1 or later with persistent connection, pipelining supported
16
< HTTP/1.1 200 OK
17
< Date: Tue, 02 Jul 2013 10:15:18 GMT
18
< Server: Apache/2.2.22 (Debian)
19
< X-Powered-By: Phusion Passenger (mod_rails/mod_rack) 3.0.13
20
< Cache-Control: no-cache
21
< X-Runtime: 133
22
< Set-Cookie: autologin=; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT
23
< Set-Cookie: _redmine_default=BAh7BzoPc2Vzc2lvbl9pZCIlOTFmNDkxMzg3MTZmNDUzYzk3NDc4NTliNjYzZmZiZDIiCmZsYXNoSUM6J0FjdGlvbkNvbnRyb2xsZXI6OkZsYXNoOjpGbGFzaEhhc2h7BjoLbm90aWNlIipNaXNlIMOgIGpvdXIgZWZmZWN0dcOpZSBhdmVjIHN1Y2PDqHMuBjoKQHVzZWR7BjsHRg%3D%3D--252700c9688855c080f30d577da1cb9e60ebc53b; path=/redmine; HttpOnly
24
< Content-Length: 1
25
< Status: 200
26
< Content-Type: application/json; charset=utf-8
27
< 
28
* Connection #0 to host localhost left intact
29
 * Closing connection #0
30
   
scripts_divers/migrer_taches_vers_redmine/purge_issues_redmine.py
1
#!/usr/env python3
2

  
3
'''Ce script supprime TOUTES les tâches redmines du projet indiqué. À utiliser avec prudence.
4
'''
5

  
6
import requests #pour faire des requêtes http
7
import json
8
import sys
9

  
10
import constantes as const
11

  
12
print('Ce script supprime TOUTES les tâches redmines du projet indiqué. À utiliser avec prudence.')
13
ok = input('Continuer ? (oui/non) : ')
14

  
15
if ok == 'oui':
16
    const.Headers_GET['X-Redmine-API-Key'] = const.SUBMITERS[const.MANAGER]
17
    r = requests.get(const.URL_ISSUES_JSON + '?status_id=*&limit=100', headers=const.Headers_GET)
18

  
19
    taches_json = json.loads(r.text)['issues']
20

  
21
    sys.stdout.write('Pourcentage de complétion : 00%\b\b\b')
22
    sys.stdout.flush()
23
    compt = 0
24
    for tache in taches_json:
25
        tid = tache['id']
26
        r = requests.delete(const.URL_REDMINE + '/issues/{}.json'.format(tid), headers=const.Headers_GET)
27

  
28
        #on affiche le pourcentage de complétion
29
        compt += 1
30
        pourcentage = compt/len(taches_json)*100
31
        sys.stdout.write(str(pourcentage) + '%\b\b\b')
32
        sys.stdout.flush()
33
else:
34
    print('Sage décision')
scripts_divers/migrer_taches_vers_redmine/url_parser.py
1
#!/usr/share/env python3
2

  
3
from html.parser import HTMLParser
4
import httplib2
5

  
6

  
7
class LinksParser(HTMLParser):
8
    "Classe permettant de parser du html"
9
    def __init__(self):
10
        HTMLParser.__init__(self)
11
        self.recording = 0
12
        self.data = []
13

  
14
    def handle_starttag(self, tag, attributes):
15
        if tag != 'span':
16
            return
17
        if self.recording:
18
            self.recording += 1
19
            return
20
        for name, value in attributes:
21
            if name == 'class' and value == 'parse_me':
22
                break
23
        else:
24
            return
25
        self.recording = 1
26

  
27
    def handle_endtag(self, tag):
28
        if tag == 'span' and self.recording:
29
            self.recording -= 1
30

  
31
    def handle_data(self, data):
32
        if self.recording:
33
            self.data.append(data)
34

  
35
def give_nids(url):
36
    p = LinksParser()
37
    h = httplib2.Http()
38

  
39
    resp, content = h.request(url, 'GET')
40
    text = content.decode('utf-8')
41

  
42
    p.feed(text)
43
    return p.data
44

  
45
def give_json_urls(url, base_url):
46
    nids = give_nids(url)
47
    tache_urls = []
48
    for nid in nids:
49
        tache_urls.append(base_url + '/node/' + nid + '.json')
50
    return nids, tache_urls #on a besoin des nids pour après.
scripts_divers/rm-prefix.pl
1
### Ce script a été écrit afin de supprimer les préfixes des tables des bases données.
2
### Ces préfixes étaient indispensables quand tous les sites étaient dans une même base de données.
3

  
4
use strict;
5
use warnings;
6
use DBI;
7

  
8
my $bd		= $ARGV[0];
9
my $serveur	= 'localhost'; # Il est possible de mettre une adresse IP
10
my $identifiant = 'root';      # identifiant
11
my $motdepasse	= 'tata';
12
my $port	= '';
13

  
14
my ($prefix) = $ARGV[1];
15

  
16
my $dbh = DBI->connect( "DBI:mysql:database=$bd;host=$serveur;port=$port", $identifiant, $motdepasse, { RaiseError => 1, } ) or die "Connection impossible à la base de données $bd !\n $! \n $@\n$DBI::errstr";
17

  
18
my @table_names = $dbh->tables;
19
foreach my $table(@table_names)
20
{
21
    #print $table =~ m/$prefix/;
22
    #print $table . "\n";
23

  
24
    if( $table =~ m/^`$bd`\.`$prefix/ )
25
    {
26
	my ($new_name) = ( $table =~ m/$prefix(.*)`/ );
27
	my $new_table = "`$bd`.`$new_name`";
28
	print $new_name . "\n";
29
	print $new_table . "\n";
30
	$dbh->do('DROP TABLE IF EXISTS ' . $new_name);
31
	$dbh->do('RENAME TABLE ' . $table . ' TO ' . $new_name) or die 'Ne peut exécuter la requête';
32
    }
33
}
34

  
35
$dbh->disconnect();

Formats disponibles : Unified diff